Skip to content

Commit

Permalink
feat(elasticsearch): added support for index time based patterns, #1034
Browse files Browse the repository at this point in the history
  • Loading branch information
@torkelo
torkelo committed Sep 6, 2015
1 parent 8db47e3 commit 0960360
Show file tree
Hide file tree
Showing 6 changed files with 114 additions and 25 deletions.
21 changes: 15 additions & 6 deletions public/app/features/org/datasourceEditCtrl.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,16 @@ function (angular, config) {

$scope.httpConfigPartialSrc = 'app/features/org/partials/datasourceHttpConfig.html';

var defaults = {
name: '',
type: 'graphite',
url: '',
access: 'proxy'
};
var defaults = {name: '', type: 'graphite', url: '', access: 'proxy' };

$scope.indexPatternTypes = [
{name: 'No pattern', value: undefined},
{name: 'Hourly', value: 'Hourly', example: '[logstash-]YYYY.MM.DD.HH'},
{name: 'Daily', value: 'Daily', example: '[logstash-]YYYY.MM.DD'},
{name: 'Weekly', value: 'Weekly', example: '[logstash-]GGGG.WW'},
{name: 'Monthly', value: 'Monthly', example: '[logstash-]YYYY.MM'},
{name: 'Yearly', value: 'Yearly', example: '[logstash-]YYYY'},
];

$scope.init = function() {
$scope.isNew = true;
Expand Down Expand Up @@ -117,6 +121,11 @@ function (angular, config) {
}
};

$scope.indexPatternTypeChanged = function() {
var def = _.findWhere($scope.indexPatternTypes, {value: $scope.current.jsonData.interval});
$scope.current.database = def.example || 'es-index-name';
};

$scope.init();

});
Expand Down
24 changes: 17 additions & 7 deletions public/app/plugins/datasource/elasticsearch/datasource.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,13 @@ define([
'angular',
'lodash',
'moment',
'kbn',
'./queryBuilder',
'./indexPattern',
'./queryCtrl',
'./directives'
],
function (angular, _, moment, ElasticQueryBuilder, IndexPattern) {
function (angular, _, moment, kbn, ElasticQueryBuilder, IndexPattern) {
'use strict';

var module = angular.module('grafana.services');
Expand All @@ -23,9 +24,9 @@ function (angular, _, moment, ElasticQueryBuilder, IndexPattern) {
this.indexPattern = new IndexPattern(datasource.index, datasource.jsonData.interval);
}

ElasticDatasource.prototype._request = function(method, url, index, data) {
ElasticDatasource.prototype._request = function(method, url, data) {
var options = {
url: this.url + "/" + index + url,
url: this.url + "/" + url,
method: method,
data: data
};
Expand All @@ -41,14 +42,14 @@ function (angular, _, moment, ElasticQueryBuilder, IndexPattern) {
};

ElasticDatasource.prototype._get = function(url) {
return this._request('GET', url, this.indexPattern.getIndexForToday())
return this._request('GET', this.indexPattern.getIndexForToday() + url)
.then(function(results) {
return results.data;
});
};

ElasticDatasource.prototype._post = function(url, data) {
return this._request('POST', url, this.index, data)
return this._request('POST', url, data)
.then(function(results) {
return results.data;
});
Expand Down Expand Up @@ -76,7 +77,7 @@ function (angular, _, moment, ElasticQueryBuilder, IndexPattern) {
"size": 10000
};

return this._request('POST', '/_search', annotation.index, data).then(function(results) {
return this._request('POST', annotation.index + '/_search', data).then(function(results) {
var list = [];
var hits = results.data.hits.hits;

Expand Down Expand Up @@ -135,11 +136,20 @@ function (angular, _, moment, ElasticQueryBuilder, IndexPattern) {
});
};

ElasticDatasource.prototype.getQueryHeader = function(timeRange) {
var header = {search_type: "count", "ignore_unavailable": true};
var from = kbn.parseDate(timeRange.from);
var to = kbn.parseDate(timeRange.to);
header.index = this.indexPattern.getIndexList(from, to);
return angular.toJson(header);
};

ElasticDatasource.prototype.query = function(options) {
var queryBuilder = new ElasticQueryBuilder();
var header = '{"index":"' + this.index + '","search_type":"count","ignore_unavailable":true}';
var payload = "";
var sentTargets = [];

var header = this.getQueryHeader(options.range);
var timeFrom = this.translateTime(options.range.from);
var timeTo = this.translateTime(options.range.to);

Expand Down
23 changes: 23 additions & 0 deletions public/app/plugins/datasource/elasticsearch/indexPattern.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,14 @@ function (_, moment) {
this.interval = interval;
}

IndexPattern.intervalMap = {
"Hours": { startOf: 'hour', amount: 'hours'},
"Daily": { startOf: 'day', amount: 'days'},
"Weekly": { startOf: 'isoWeek', amount: 'weeks'},
"Monthly": { startOf: 'month', amount: 'months'},
"Yearly": { startOf: 'year', amount: 'years'},
};

IndexPattern.prototype.getIndexForToday = function() {
if (this.interval) {
return moment().format(this.pattern);
Expand All @@ -19,6 +27,21 @@ function (_, moment) {
};

IndexPattern.prototype.getIndexList = function(from, to) {
if (!this.interval) {
return this.pattern;
}

var intervalInfo = IndexPattern.intervalMap[this.interval];
var start = moment(from).utc().startOf(intervalInfo.startOf);
var end = moment(to).utc().startOf(intervalInfo.startOf).valueOf();
var indexList = [];

while (start <= end) {
indexList.push(start.format(this.pattern));
start.add(1, intervalInfo.amount);
}

return indexList;
};

return IndexPattern;
Expand Down
3 changes: 1 addition & 2 deletions public/app/plugins/datasource/elasticsearch/partials/config.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,7 @@ <h5>Elastic search details</h5>
Pattern
</li>
<li>
<select class="input-medium tight-form-input" ng-model="current.jsonData.interval"
ng-options="f.value as f.name for f in [{name: 'No pattern', value: undefined}, {name: 'Hourly', value: 'hourly'}, {name: 'Daily', value: 'daily'}]" ></select>
<select class="input-medium tight-form-input" ng-model="current.jsonData.interval" ng-options="f.value as f.name for f in indexPatternTypes" ng-change="indexPatternTypeChanged()" ></select>
</li>
</ul>
<div class="clearfix"></div>
Expand Down
31 changes: 24 additions & 7 deletions public/test/specs/elasticsearch-indexPattern-specs.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ define([

describe('when getting index for today', function() {
it('should return correct index name', function() {
var pattern = new IndexPattern('[asd-]YYYY.MM.DD', 'daily');
var pattern = new IndexPattern('[asd-]YYYY.MM.DD', 'Daily');
var expected = 'asd-' + moment().format('YYYY.MM.DD');

expect(pattern.getIndexForToday()).to.be(expected);
Expand All @@ -17,16 +17,33 @@ define([

describe('when getting index list for time range', function() {

describe('no interval', function() {
it('should return correct index', function() {
var pattern = new IndexPattern('my-metrics');
var from = new Date(2015, 4, 30, 1, 2, 3);
var to = new Date(2015, 5, 1, 12, 5 , 6);
expect(pattern.getIndexList(from, to)).to.eql('my-metrics');
});
});

describe('daily', function() {

it('should return correct index list', function() {
var pattern = new IndexPattern('[asd-]YYYY.MM.DD', 'daily');
var from = new Date(2015, 4, 29);
var to = new Date(2015, 5, 1);

expect(pattern.getIndexList(from, to)).to.be(['asd', 'asd2']);
var pattern = new IndexPattern('[asd-]YYYY.MM.DD', 'Daily');
var from = new Date(2015, 4, 30, 1, 2, 3);
var to = new Date(2015, 5, 1, 12, 5 , 6);

var expected = [
'asd-2015.05.29',
'asd-2015.05.30',
'asd-2015.05.31',
'asd-2015.06.01',
];

expect(pattern.getIndexList(from, to)).to.eql(expected);
});
})

});

});

Expand Down
37 changes: 34 additions & 3 deletions public/test/specs/elasticsearch-specs.js
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
define([
'helpers',
'moment',
'angular',
'plugins/datasource/elasticsearch/datasource',
'aws-sdk',
], function(helpers, moment) {
], function(helpers, moment, angular) {
'use strict';

describe('ElasticDatasource', function() {
Expand All @@ -21,9 +22,9 @@ define([
ctx.ds = new ctx.service({
url: 'http://es.com',
index: '[asd-]YYYY.MM.DD',
jsonData: { interval: 'daily' }
jsonData: { interval: 'Daily' }
});
})
});

it('should translate index pattern to current day', function() {
var requestOptions;
Expand All @@ -38,7 +39,37 @@ define([
var today = moment().format("YYYY.MM.DD");
expect(requestOptions.url).to.be("http://es.com/asd-" + today + '/_stats');
});
});

describe('When issueing metric query with interval pattern', function() {
beforeEach(function() {
ctx.ds = new ctx.service({
url: 'http://es.com',
index: '[asd-]YYYY.MM.DD',
jsonData: { interval: 'Daily' }
});
});

it('should translate index pattern to current day', function() {
var requestOptions;
ctx.backendSrv.datasourceRequest = function(options) {
requestOptions = options;
return ctx.$q.when({data: {responses: []}});
};

ctx.ds.query({
range: {
from: new Date(2015, 4, 30, 10),
to: new Date(2015, 5, 1, 10)
},
targets: [{ bucketAggs: [], metrics: [] }]
});

ctx.$rootScope.$apply();
var parts = requestOptions.data.split('\n');
var header = angular.fromJson(parts[0]);
expect(header.index).to.eql(['asd-2015.05.30', 'asd-2015.05.31', 'asd-2015.06.01']);
});
});

describe('When processing es response', function() {
Expand Down

0 comments on commit 0960360

Please sign in to comment.