Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Allow for backwards compatibility for unix timestamp in pre 2.x indices #11515

Merged
merged 1 commit into from Jun 25, 2015
Merged

Allow for backwards compatibility for unix timestamp in pre 2.x indices #11515

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
11 changes: 8 additions & 3 deletions core/src/main/java/org/elasticsearch/action/index/IndexRequest.java
View file
Open in desktop
Expand Up @@ -22,6 +22,7 @@
import com.google.common.base.Charsets;
import org.elasticsearch.ElasticsearchGenerationException;
import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.Version;
import org.elasticsearch.action.*;
import org.elasticsearch.action.support.replication.ReplicationRequest;
import org.elasticsearch.client.Requests;
Expand Down Expand Up @@ -562,8 +563,10 @@ public void process(MetaData metaData, @Nullable MappingMetaData mappingMd, bool
routing(metaData.resolveIndexRouting(routing, index));
// resolve timestamp if provided externally
if (timestamp != null) {
Version version = Version.indexCreated(metaData.getIndices().get(concreteIndex).settings());
timestamp = MappingMetaData.Timestamp.parseStringTimestamp(timestamp,
mappingMd != null ? mappingMd.timestamp().dateTimeFormatter() : TimestampFieldMapper.Defaults.DATE_TIME_FORMATTER);
mappingMd != null ? mappingMd.timestamp().dateTimeFormatter() : TimestampFieldMapper.Defaults.DATE_TIME_FORMATTER,
version);
}
// extract values if needed
if (mappingMd != null) {
Expand All @@ -586,7 +589,8 @@ public void process(MetaData metaData, @Nullable MappingMetaData mappingMd, bool
if (parseContext.shouldParseTimestamp()) {
timestamp = parseContext.timestamp();
if (timestamp != null) {
timestamp = MappingMetaData.Timestamp.parseStringTimestamp(timestamp, mappingMd.timestamp().dateTimeFormatter());
Version version = Version.indexCreated(metaData.getIndices().get(concreteIndex).settings());
timestamp = MappingMetaData.Timestamp.parseStringTimestamp(timestamp, mappingMd.timestamp().dateTimeFormatter(), version);
}
}
} catch (MapperParsingException e) {
Expand Down Expand Up @@ -638,7 +642,8 @@ public void process(MetaData metaData, @Nullable MappingMetaData mappingMd, bool
if (defaultTimestamp.equals(TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP)) {
timestamp = Long.toString(System.currentTimeMillis());
} else {
timestamp = MappingMetaData.Timestamp.parseStringTimestamp(defaultTimestamp, mappingMd.timestamp().dateTimeFormatter());
Version version = Version.indexCreated(metaData.getIndices().get(concreteIndex).settings());
timestamp = MappingMetaData.Timestamp.parseStringTimestamp(defaultTimestamp, mappingMd.timestamp().dateTimeFormatter(), version);
}
}
}
Expand Down
17 changes: 15 additions & 2 deletions core/src/main/java/org/elasticsearch/cluster/metadata/MappingMetaData.java
View file
Open in desktop
Expand Up @@ -19,6 +19,7 @@

package org.elasticsearch.cluster.metadata;

import org.elasticsearch.Version;
import org.elasticsearch.action.TimestampParsingException;
import org.elasticsearch.cluster.AbstractDiffable;
import org.elasticsearch.common.Nullable;
Expand Down Expand Up @@ -160,10 +161,22 @@ public int hashCode() {

public static class Timestamp {

public static String parseStringTimestamp(String timestampAsString, FormatDateTimeFormatter dateTimeFormatter) throws TimestampParsingException {
private static final FormatDateTimeFormatter EPOCH_MILLIS_PARSER = Joda.forPattern("epoch_millis");

public static String parseStringTimestamp(String timestampAsString, FormatDateTimeFormatter dateTimeFormatter,
Version version) throws TimestampParsingException {
try {
return Long.toString(dateTimeFormatter.parser().parseMillis(timestampAsString));
// no need for unix timestamp parsing in 2.x
FormatDateTimeFormatter formatter = version.onOrAfter(Version.V_2_0_0) ? dateTimeFormatter : EPOCH_MILLIS_PARSER;
return Long.toString(formatter.parser().parseMillis(timestampAsString));
} catch (RuntimeException e) {
if (version.before(Version.V_2_0_0)) {
try {
return Long.toString(dateTimeFormatter.parser().parseMillis(timestampAsString));
} catch (RuntimeException e1) {
throw new TimestampParsingException(timestampAsString, e1);
}
}
throw new TimestampParsingException(timestampAsString, e);
}
}
Expand Down
32 changes: 16 additions & 16 deletions core/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java
View file
Open in desktop
Expand Up @@ -30,6 +30,7 @@
import org.apache.lucene.util.BytesRefBuilder;
import org.apache.lucene.util.NumericUtils;
import org.apache.lucene.util.ToStringUtils;
import org.elasticsearch.Version;
import org.elasticsearch.action.fieldstats.FieldStats;
import org.elasticsearch.common.Explicit;
import org.elasticsearch.common.Nullable;
Expand All @@ -46,18 +47,17 @@
import org.elasticsearch.index.analysis.NamedAnalyzer;
import org.elasticsearch.index.analysis.NumericDateAnalyzer;
import org.elasticsearch.index.fielddata.FieldDataType;
import org.elasticsearch.index.mapper.*;
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.index.mapper.Mapper;
import org.elasticsearch.index.mapper.MapperParsingException;
import org.elasticsearch.index.mapper.ParseContext;
import org.elasticsearch.index.mapper.core.LongFieldMapper.CustomLongNumericField;
import org.elasticsearch.index.query.QueryParseContext;
import org.elasticsearch.search.internal.SearchContext;
import org.joda.time.DateTimeZone;

import java.io.IOException;
import java.util.Iterator;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.*;
import java.util.concurrent.Callable;
import java.util.concurrent.TimeUnit;

Expand All @@ -70,7 +70,7 @@ public class DateFieldMapper extends NumberFieldMapper {
public static final String CONTENT_TYPE = "date";

public static class Defaults extends NumberFieldMapper.Defaults {
public static final FormatDateTimeFormatter DATE_TIME_FORMATTER = Joda.forPattern("dateOptionalTime", Locale.ROOT);
public static final FormatDateTimeFormatter DATE_TIME_FORMATTER = Joda.forPattern("dateOptionalTime||epoch_millis", Locale.ROOT);
public static final TimeUnit TIME_UNIT = TimeUnit.MILLISECONDS;
public static final DateFieldType FIELD_TYPE = new DateFieldType();

Expand Down Expand Up @@ -126,6 +126,14 @@ public DateFieldMapper build(BuilderContext context) {

protected void setupFieldType(BuilderContext context) {
FormatDateTimeFormatter dateTimeFormatter = fieldType().dateTimeFormatter;
// TODO MOVE ME OUTSIDE OF THIS SPACE?
if (Version.indexCreated(context.indexSettings()).before(Version.V_2_0_0)) {
boolean includesEpochFormatter = dateTimeFormatter.format().contains("epoch_");
if (!includesEpochFormatter) {
String format = fieldType().timeUnit().equals(TimeUnit.SECONDS) ? "epoch_second" : "epoch_millis";
fieldType().setDateTimeFormatter(Joda.forPattern(format + "||" + dateTimeFormatter.format()));
}
}
if (!locale.equals(dateTimeFormatter.locale())) {
fieldType().setDateTimeFormatter(new FormatDateTimeFormatter(dateTimeFormatter.format(), dateTimeFormatter.parser(), dateTimeFormatter.printer(), locale));
}
Expand Down Expand Up @@ -308,15 +316,7 @@ private long parseValue(Object value) {
}

protected long parseStringValue(String value) {
try {
return dateTimeFormatter().parser().parseMillis(value);
} catch (RuntimeException e) {
try {
return timeUnit().toMillis(Long.parseLong(value));
} catch (NumberFormatException e1) {
throw new MapperParsingException("failed to parse date field [" + value + "], tried both date format [" + dateTimeFormatter().format() + "], and timestamp number with locale [" + dateTimeFormatter().locale() + "]", e);
}
}
return dateTimeFormatter().parser().parseMillis(value);
}

@Override
Expand Down
3 changes: 3 additions & 0 deletions core/src/main/java/org/elasticsearch/index/mapper/object/RootObjectMapper.java
View file
Open in desktop
Expand Up @@ -147,6 +147,9 @@ protected boolean processField(ObjectMapper.Builder builder, String fieldName, O
List<FormatDateTimeFormatter> dateTimeFormatters = newArrayList();
if (fieldNode instanceof List) {
for (Object node1 : (List) fieldNode) {
if (node1.toString().startsWith("epoch_")) {
throw new MapperParsingException("Epoch ["+ node1.toString() +"] is not supported as dynamic date format");
}
dateTimeFormatters.add(parseDateTimeFormatter(node1));
}
} else if ("none".equals(fieldNode.toString())) {
Expand Down
18 changes: 12 additions & 6 deletions core/src/main/java/org/elasticsearch/index/query/RangeQueryBuilder.java
View file
Open in desktop
Expand Up @@ -29,19 +29,14 @@
public class RangeQueryBuilder extends MultiTermQueryBuilder implements BoostableQueryBuilder<RangeQueryBuilder> {

private final String name;

private Object from;

private Object to;
private String timeZone;

private boolean includeLower = true;

private boolean includeUpper = true;

private float boost = -1;

private String queryName;
private String format;

/**
* A Query that matches documents within an range of terms.
Expand Down Expand Up @@ -406,6 +401,14 @@ public RangeQueryBuilder timeZone(String timezone) {
return this;
}

/**
* In case of date field, we can set the format to be used instead of the mapper format
*/
public RangeQueryBuilder format(String format) {
this.format = format;
return this;
}

@Override
protected void doXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject(RangeQueryParser.NAME);
Expand All @@ -415,6 +418,9 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep
if (timeZone != null) {
builder.field("time_zone", timeZone);
}
if (format != null) {
builder.field("format", format);
}
builder.field("include_lower", includeLower);
builder.field("include_upper", includeUpper);
if (boost != -1) {
Expand Down
11 changes: 2 additions & 9 deletions core/src/test/java/org/elasticsearch/count/simple/SimpleCountTests.java
View file
Open in desktop
Expand Up @@ -181,7 +181,7 @@ public void localDependentDateTests() throws Exception {
}

@Test
public void testThatNonEpochDatesCanBeSearch() throws Exception {
public void testThatNonEpochDatesCanBeSearched() throws Exception {
assertAcked(prepareCreate("test")
.addMapping("type1",
jsonBuilder().startObject().startObject("type1")
Expand All @@ -201,16 +201,9 @@ public void testThatNonEpochDatesCanBeSearch() throws Exception {
.endObject();
assertThat(client().prepareIndex("test", "type1").setSource(document).get().isCreated(), is(true));

// this is a timestamp in 2015 and should not be returned in counting when filtering by year
document = jsonBuilder()
.startObject()
.field("date_field", "1433236702")
.endObject();
assertThat(client().prepareIndex("test", "type1").setSource(document).get().isCreated(), is(true));

refresh();

assertHitCount(client().prepareCount("test").get(), 3);
assertHitCount(client().prepareCount("test").get(), 2);

CountResponse countResponse = client().prepareCount("test").setQuery(QueryBuilders.rangeQuery("date_field").from("2015010100").to("2015123123")).get();
assertHitCount(countResponse, 1);
Expand Down