Skip to content

Commit

Permalink
Implement simple query time range limit
Browse files Browse the repository at this point in the history
In order to guard the system against accidental queries covering a large
time range, the "query_time_range_limit" setting now provides a limit for
those queries.
If a search query exceeeds the time range given in "query_time_range_limit",
the query is modified to fit within that limit by adjusting the starting time.
  • Loading branch information
Jochen Schalanda committed Feb 12, 2016
1 parent b202265 commit 821fcde
Show file tree
Hide file tree
Showing 9 changed files with 170 additions and 13 deletions.
9 changes: 9 additions & 0 deletions graylog2-server/src/main/java/org/graylog2/Configuration.java
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import org.graylog2.plugin.BaseConfiguration;
import org.joda.time.DateTimeZone;

import javax.annotation.Nullable;
import java.net.URI;
import java.nio.file.Path;
import java.nio.file.Paths;
Expand Down Expand Up @@ -158,6 +159,9 @@ public class Configuration extends BaseConfiguration {
@Parameter(value = "index_ranges_cleanup_interval", validator = PositiveDurationValidator.class)
private Duration indexRangesCleanupInterval = Duration.hours(1L);

@Parameter(value = "query_time_range_limit")
private org.joda.time.Duration queryTimeRangeLimit = null;

public boolean isMaster() {
return isMaster;
}
Expand Down Expand Up @@ -319,4 +323,9 @@ public Set<String> getContentPacksAutoLoad() {
public Duration getIndexRangesCleanupInterval() {
return indexRangesCleanupInterval;
}

@Nullable
public org.joda.time.Duration getQueryTimeRangeLimit() {
return queryTimeRangeLimit;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -40,10 +40,13 @@
import org.graylog2.shared.rest.AdditionalMediaType;
import org.graylog2.shared.security.RestPermissions;
import org.hibernate.validator.constraints.NotEmpty;
import org.joda.time.Duration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.annotation.Nullable;
import javax.inject.Inject;
import javax.inject.Named;
import javax.ws.rs.BadRequestException;
import javax.ws.rs.GET;
import javax.ws.rs.Path;
Expand All @@ -60,8 +63,8 @@ public class AbsoluteSearchResource extends SearchResource {
private static final Logger LOG = LoggerFactory.getLogger(AbsoluteSearchResource.class);

@Inject
public AbsoluteSearchResource(Searches searches) {
super(searches);
public AbsoluteSearchResource(Searches searches, @Named("query_time_range_limit") @Nullable Duration timeRangeLimit) {
super(searches, timeRangeLimit);
}

@GET
Expand Down Expand Up @@ -310,7 +313,7 @@ public HistogramResult fieldHistogramAbsolute(

private TimeRange buildAbsoluteTimeRange(String from, String to) {
try {
return AbsoluteRange.create(from, to);
return restrictTimeRange(AbsoluteRange.create(from, to));
} catch (InvalidRangeParametersException e) {
LOG.warn("Invalid timerange parameters provided. Returning HTTP 400.");
throw new BadRequestException("Invalid timerange parameters provided", e);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,10 +40,13 @@
import org.graylog2.shared.rest.AdditionalMediaType;
import org.graylog2.shared.security.RestPermissions;
import org.hibernate.validator.constraints.NotEmpty;
import org.joda.time.Duration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.annotation.Nullable;
import javax.inject.Inject;
import javax.inject.Named;
import javax.ws.rs.BadRequestException;
import javax.ws.rs.GET;
import javax.ws.rs.Path;
Expand All @@ -61,8 +64,8 @@ public class KeywordSearchResource extends SearchResource {
private static final Logger LOG = LoggerFactory.getLogger(KeywordSearchResource.class);

@Inject
public KeywordSearchResource(Searches searches) {
super(searches);
public KeywordSearchResource(Searches searches, @Named("query_time_range_limit") @Nullable Duration timeRangeLimit) {
super(searches, timeRangeLimit);
}

@GET
Expand Down Expand Up @@ -295,7 +298,7 @@ public HistogramResult fieldHistogramKeyword(

private TimeRange buildKeywordTimeRange(String keyword) {
try {
return KeywordRange.create(keyword);
return restrictTimeRange(KeywordRange.create(keyword));
} catch (InvalidRangeParametersException e) {
LOG.warn("Invalid timerange parameters provided. Returning HTTP 400.");
throw new BadRequestException("Invalid timerange parameters provided", e);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,10 +40,13 @@
import org.graylog2.shared.rest.AdditionalMediaType;
import org.graylog2.shared.security.RestPermissions;
import org.hibernate.validator.constraints.NotEmpty;
import org.joda.time.Duration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.annotation.Nullable;
import javax.inject.Inject;
import javax.inject.Named;
import javax.ws.rs.BadRequestException;
import javax.ws.rs.GET;
import javax.ws.rs.Path;
Expand All @@ -61,8 +64,8 @@ public class RelativeSearchResource extends SearchResource {
private static final Logger LOG = LoggerFactory.getLogger(RelativeSearchResource.class);

@Inject
public RelativeSearchResource(Searches searches) {
super(searches);
public RelativeSearchResource(Searches searches, @Named("query_time_range_limit") @Nullable Duration timeRangeLimit) {
super(searches, timeRangeLimit);
}

@GET
Expand Down Expand Up @@ -297,7 +300,7 @@ public HistogramResult fieldHistogramRelative(

private TimeRange buildRelativeTimeRange(int range) {
try {
return RelativeRange.create(range);
return restrictTimeRange(RelativeRange.create(range));
} catch (InvalidRangeParametersException e) {
LOG.warn("Invalid timerange parameters provided. Returning HTTP 400.");
throw new BadRequestException(e);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,11 @@
import org.graylog2.savedsearches.SavedSearch;
import org.graylog2.savedsearches.SavedSearchService;
import org.graylog2.shared.security.RestPermissions;
import org.joda.time.Duration;

import javax.annotation.Nullable;
import javax.inject.Inject;
import javax.inject.Named;
import javax.validation.Valid;
import javax.ws.rs.BadRequestException;
import javax.ws.rs.Consumes;
Expand All @@ -60,8 +63,9 @@ public class SavedSearchesResource extends SearchResource {

@Inject
public SavedSearchesResource(Searches searches,
SavedSearchService savedSearchService) {
super(searches);
SavedSearchService savedSearchService,
@Named("query_time_range_limit") @Nullable Duration timeRangeLimit) {
super(searches, timeRangeLimit);
this.savedSearchService = savedSearchService;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,10 +45,14 @@
import org.graylog2.shared.rest.resources.RestResource;
import org.graylog2.shared.security.RestPermissions;
import org.graylog2.shared.utilities.ExceptionUtils;
import org.joda.time.DateTime;
import org.joda.time.Duration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.annotation.Nullable;
import javax.inject.Inject;
import javax.inject.Named;
import javax.ws.rs.BadRequestException;
import javax.ws.rs.ForbiddenException;
import javax.ws.rs.core.Response;
Expand All @@ -64,10 +68,12 @@ public abstract class SearchResource extends RestResource {
private static final Logger LOG = LoggerFactory.getLogger(SearchResource.class);

protected final Searches searches;
protected final Duration timeRangeLimit;

@Inject
public SearchResource(Searches searches) {
public SearchResource(Searches searches, @Named("query_time_range_limit") @Nullable Duration timeRangeLimit) {
this.searches = searches;
this.timeRangeLimit = timeRangeLimit;
}

protected void validateInterval(String interval) {
Expand Down Expand Up @@ -223,7 +229,7 @@ protected BadRequestException createRequestExceptionForParseFailure(String query
QueryParseError errorMessage = QueryParseError.create(query, "Unable to execute search", e.getClass().getCanonicalName());

// We're so going to hell for this…
if(e.getMessage().contains("nested: ParseException")) {
if (e.getMessage().contains("nested: ParseException")) {
final QueryParser queryParser = new QueryParser("", new StandardAnalyzer());
try {
queryParser.parse(query);
Expand Down Expand Up @@ -332,4 +338,18 @@ public void run() {
}
};
}

protected org.graylog2.indexer.searches.timeranges.TimeRange restrictTimeRange(final org.graylog2.indexer.searches.timeranges.TimeRange timeRange) {
final DateTime originalFrom = timeRange.getFrom();
final DateTime to = timeRange.getTo();
final DateTime from;
if (timeRangeLimit == null) {
from = originalFrom;
} else {
final DateTime limitedFrom = to.minus(timeRangeLimit);
from = limitedFrom.isAfter(originalFrom) ? limitedFrom : originalFrom;
}

return AbsoluteRange.create(from, to);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -18,16 +18,19 @@

import com.google.common.collect.ImmutableSet;
import com.google.common.collect.ImmutableSortedSet;
import org.elasticsearch.index.query.RangeQueryBuilder;
import org.graylog2.indexer.ranges.IndexRange;
import org.graylog2.indexer.ranges.IndexRangeService;
import org.graylog2.indexer.ranges.MongoIndexRange;
import org.graylog2.indexer.searches.timeranges.AbsoluteRange;
import org.graylog2.indexer.searches.timeranges.KeywordRange;
import org.graylog2.indexer.searches.timeranges.RelativeRange;
import org.graylog2.indexer.searches.timeranges.TimeRange;
import org.graylog2.plugin.Tools;
import org.joda.time.DateTime;
import org.joda.time.DateTimeUtils;
import org.joda.time.DateTimeZone;
import org.joda.time.Duration;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Rule;
Expand Down Expand Up @@ -202,4 +205,22 @@ public void determineAffectedIndicesDoesNotIncludesDeflectorTargetIfMissing() th
assertThat(IndexHelper.determineAffectedIndices(indexRangeService, deflector, relativeRange))
.containsOnly(indexRange0.indexName(), indexRange1.indexName());
}

@Test
public void getTimestampRangeFilterReturnsNullIfTimeRangeIsNull() {
assertThat(IndexHelper.getTimestampRangeFilter(null)).isNull();
}

@Test
public void getTimestampRangeFilterReturnsRangeQueryWithGivenTimeRange() {
final DateTime from = new DateTime(2016, 1, 15, 12, 0, DateTimeZone.UTC);
final DateTime to = from.plusHours(1);
final TimeRange timeRange = AbsoluteRange.create(from, to);
final RangeQueryBuilder queryBuilder = (RangeQueryBuilder) IndexHelper.getTimestampRangeFilter(timeRange);
assertThat(queryBuilder)
.isNotNull()
.hasFieldOrPropertyWithValue("name", "timestamp")
.hasFieldOrPropertyWithValue("from", Tools.buildElasticSearchTimeFormat(from))
.hasFieldOrPropertyWithValue("to", Tools.buildElasticSearchTimeFormat(to));
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
/**
* This file is part of Graylog.
*
* Graylog is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Graylog is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with Graylog. If not, see <http://www.gnu.org/licenses/>.
*/
package org.graylog2.rest.resources.search;

import org.graylog2.indexer.searches.Searches;
import org.graylog2.indexer.searches.timeranges.AbsoluteRange;
import org.graylog2.indexer.searches.timeranges.TimeRange;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
import org.joda.time.Duration;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.mockito.Mock;
import org.mockito.junit.MockitoJUnit;
import org.mockito.junit.MockitoRule;

import static org.assertj.core.api.Assertions.assertThat;

public class SearchResourceTest {
@Rule
public final MockitoRule mockitoRule = MockitoJUnit.rule();

@Mock
private Searches searches;

private SearchResource searchResource;
private Duration timeRangeLimit;

@Before
public void setUp() {
timeRangeLimit = Duration.standardDays(1L);
searchResource = new SearchResource(searches, timeRangeLimit) {
};
}

@Test
public void restrictTimeRangeReturnsGivenTimeRangeWithinLimit() {
final DateTime from = new DateTime(2015, 1, 15, 12, 0, DateTimeZone.UTC);
final DateTime to = from.plusHours(1);
final TimeRange timeRange = AbsoluteRange.create(from, to);

final TimeRange restrictedTimeRange = searchResource.restrictTimeRange(timeRange);
assertThat(restrictedTimeRange).isNotNull();
assertThat(restrictedTimeRange.getFrom()).isEqualTo(from);
assertThat(restrictedTimeRange.getTo()).isEqualTo(to);
}

@Test
public void restrictTimeRangeReturnsGivenTimeRangeIfNoLimitHasBeenSet() {
final SearchResource resource = new SearchResource(searches, null) {
};

final DateTime from = new DateTime(2015, 1, 15, 12, 0, DateTimeZone.UTC);
final DateTime to = from.plusYears(1);
final TimeRange timeRange = AbsoluteRange.create(from, to);

final TimeRange restrictedTimeRange = resource.restrictTimeRange(timeRange);
assertThat(restrictedTimeRange).isNotNull();
assertThat(restrictedTimeRange.getFrom()).isEqualTo(from);
assertThat(restrictedTimeRange.getTo()).isEqualTo(to);
}

@Test
public void restrictTimeRangeReturnsLimitedTimeRange() {
final DateTime from = new DateTime(2015, 1, 15, 12, 0, DateTimeZone.UTC);
final DateTime to = from.plus(timeRangeLimit.multipliedBy(2L));
final TimeRange timeRange = AbsoluteRange.create(from, to);
final TimeRange restrictedTimeRange = searchResource.restrictTimeRange(timeRange);

assertThat(restrictedTimeRange).isNotNull();
assertThat(restrictedTimeRange.getFrom()).isEqualTo(to.minus(timeRangeLimit));
assertThat(restrictedTimeRange.getTo()).isEqualTo(to);
}
}
5 changes: 5 additions & 0 deletions misc/graylog.conf
Original file line number Diff line number Diff line change
Expand Up @@ -228,6 +228,11 @@ elasticsearch_analyzer = standard
# Default: 1m
#elasticsearch_request_timeout = 1m

# The maximum time users can query data in the past. This prevents users from accidentally creating queries which
# span a lot of data and would need a long time and many resources to complete (if at all).
# Default: none
#query_time_range_limit = 30d

# Time interval for index range information cleanups. This setting defines how often stale index range information
# is being purged from the database.
# Default: 1h
Expand Down

0 comments on commit 821fcde

Please sign in to comment.