Skip to content

Commit

Permalink
Index name expression resolver bwc layer for date parsing (#58503)
Browse files Browse the repository at this point in the history 
backports #34507

closes #58481
also closes general parsing issue (no date processors) #58602
  • Loading branch information
@pgomulka
pgomulka committed Jul 2, 2020
1 parent fd5a708 commit e49df55
Show file tree
Hide file tree
Showing 4 changed files with 141 additions and 48 deletions.
63 changes: 63 additions & 0 deletions ...est-common/src/test/resources/rest-api-spec/test/ingest/100_date_index_name_processor.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,3 +34,66 @@ teardown:
date: "2016-04-22T16:32:14.968Z"
}
- match: { _index: "events-2016-04-22"}

---
"Test date index name processor with joda pattern":
- do:
ingest.put_pipeline:
id: "1"
body: >
{
"processors": [
{
"date_index_name" : {
"field": "date",
"date_rounding": "d",
"index_name_prefix": "prefix-",
"index_name_format": "xxxx-w"
}
}
]
}
- match: { acknowledged: true }

- do:
index:
index: test
type: _doc
id: 1
pipeline: "1"
body: {
date: "2020-08-10T01:01:01.000Z"
}
- match: { _index: "prefix-2020-33"}


---
"Test date index name processor with java pattern":
- do:
ingest.put_pipeline:
id: "1"
body: >
{
"processors": [
{
"date_index_name" : {
"field": "date",
"date_rounding": "d",
"index_name_prefix": "prefix-",
"index_name_format": "8YYYY-w"
}
}
]
}
- match: { acknowledged: true }

- do:
index:
index: test
type: _doc
id: 1
pipeline: "1"
body: {
date: "2020-08-10T01:01:01.000Z"
}
- match: { _index: "prefix-2020-33"}
52 changes: 52 additions & 0 deletions rest-api-spec/src/main/resources/rest-api-spec/test/index/81_week_based_java_date.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
---
"Index java 8 date without timezone":

- skip:
version: " - 6.7.99"
reason: fixed in 6.8.11

- do:
indices.create:
index: test_index
body:
settings:
number_of_shards: 1
mappings:
doc:
properties:
date_field:
type: date
format: "8YYYY-ww"
- do:
bulk:
refresh: true
body:
- '{"index": {"_index": "test_index", "_type": "doc", "_id": "1"}}'
- '{"date_field": "2020-32"}'
- '{"index": {"_index": "test_index", "_type": "doc", "_id": "2"}}'
- '{"date_field": "2020-33"}'

- match: { errors: false }

- do:
get:
index: test_index
type: doc
id: 2

- match: { _index: test_index }
- match: { _type: doc }
- match: { _id: "2"}
- match: { _version: 1}
- match: { _source: { "date_field": "2020-33" }}

- do:
search:
index: test_index
body: { "query": { "range": { "date_field": { "gte": "2020-33" } } } }

- match: {hits.total: 1 }
- match: {hits.hits.0._index: test_index }
- match: {hits.hits.0._type: doc }
- match: {hits.hits.0._source.date_field: "2020-33" }

67 changes: 26 additions & 41 deletions server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,29 +27,26 @@
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.joda.JodaDateFormatter;
import org.elasticsearch.common.regex.Regex;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.time.DateFormatter;
import org.elasticsearch.common.time.DateFormatters;
import org.elasticsearch.common.time.DateMathParser;
import org.elasticsearch.common.time.DateUtils;
import org.elasticsearch.common.util.set.Sets;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.IndexNotFoundException;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.indices.IndexClosedException;
import org.elasticsearch.indices.InvalidIndexNameException;
import org.joda.time.DateTimeZone;
import org.joda.time.format.DateTimeFormat;
import org.joda.time.format.DateTimeFormatter;

import java.time.ZoneId;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
Expand All @@ -66,10 +63,10 @@ public class IndexNameExpressionResolver {
public IndexNameExpressionResolver(Settings settings) {
dateMathExpressionResolver = new DateMathExpressionResolver(settings);
expressionResolvers = Arrays.asList(
dateMathExpressionResolver,
new WildcardExpressionResolver());
dateMathExpressionResolver,
new WildcardExpressionResolver()
);
}


/**
* Same as {@link #concreteIndexNames(ClusterState, IndicesOptions, String...)}, but the index expressions and options
Expand Down Expand Up @@ -848,22 +845,23 @@ private static List<String> resolveEmptyOrTrivialWildcard(IndicesOptions options

static final class DateMathExpressionResolver implements ExpressionResolver {

private static final DateFormatter DEFAULT_DATE_FORMATTER = DateFormatters.forPattern("uuuu.MM.dd");
private static final String EXPRESSION_LEFT_BOUND = "<";
private static final String EXPRESSION_RIGHT_BOUND = ">";
private static final char LEFT_BOUND = '{';
private static final char RIGHT_BOUND = '}';
private static final char ESCAPE_CHAR = '\\';
private static final char TIME_ZONE_BOUND = '|';

private final DateTimeZone defaultTimeZone;
private final ZoneId defaultTimeZone;
private final String defaultDateFormatterPattern;
private final DateTimeFormatter defaultDateFormatter;
private final DateFormatter defaultDateFormatter;

DateMathExpressionResolver(Settings settings) {
String defaultTimeZoneId = settings.get("date_math_expression_resolver.default_time_zone", "UTC");
this.defaultTimeZone = DateTimeZone.forID(defaultTimeZoneId);
defaultDateFormatterPattern = settings.get("date_math_expression_resolver.default_date_format", "YYYY.MM.dd");
this.defaultDateFormatter = DateTimeFormat.forPattern(defaultDateFormatterPattern);
this.defaultTimeZone = ZoneId.of(defaultTimeZoneId);
defaultDateFormatterPattern = settings.get("date_math_expression_resolver.default_date_format", "8uuuu.MM.dd");
this.defaultDateFormatter = DateFormatter.forPattern(defaultDateFormatterPattern);
}

@Override
Expand Down Expand Up @@ -930,11 +928,10 @@ String resolveExpression(String expression, final Context context) {
int dateTimeFormatLeftBoundIndex = inPlaceHolderString.indexOf(LEFT_BOUND);
String mathExpression;
String dateFormatterPattern;
DateTimeFormatter dateFormatter;
final DateTimeZone timeZone;
DateFormatter dateFormatter;
final ZoneId timeZone;
if (dateTimeFormatLeftBoundIndex < 0) {
mathExpression = inPlaceHolderString;
dateFormatterPattern = defaultDateFormatterPattern;
dateFormatter = defaultDateFormatter;
timeZone = defaultTimeZone;
} else {
Expand All @@ -947,23 +944,25 @@ String resolveExpression(String expression, final Context context) {
inPlaceHolderString);
}
mathExpression = inPlaceHolderString.substring(0, dateTimeFormatLeftBoundIndex);
String patternAndTZid =
String dateFormatterPatternAndTimeZoneId =
inPlaceHolderString.substring(dateTimeFormatLeftBoundIndex + 1, inPlaceHolderString.length() - 1);
int formatPatternTimeZoneSeparatorIndex = patternAndTZid.indexOf(TIME_ZONE_BOUND);
int formatPatternTimeZoneSeparatorIndex = dateFormatterPatternAndTimeZoneId.indexOf(TIME_ZONE_BOUND);
if (formatPatternTimeZoneSeparatorIndex != -1) {
dateFormatterPattern = patternAndTZid.substring(0, formatPatternTimeZoneSeparatorIndex);
timeZone = DateTimeZone.forID(patternAndTZid.substring(formatPatternTimeZoneSeparatorIndex + 1));
dateFormatterPattern
= dateFormatterPatternAndTimeZoneId.substring(0, formatPatternTimeZoneSeparatorIndex);
timeZone = ZoneId.of(
dateFormatterPatternAndTimeZoneId.substring(formatPatternTimeZoneSeparatorIndex + 1));
} else {
dateFormatterPattern = patternAndTZid;
dateFormatterPattern = dateFormatterPatternAndTimeZoneId;
timeZone = defaultTimeZone;
}
dateFormatter = DateTimeFormat.forPattern(dateFormatterPattern);
dateFormatter = DateFormatter.forPattern(dateFormatterPattern);
}
DateTimeFormatter parser = dateFormatter.withLocale(Locale.ROOT).withZone(timeZone);
JodaDateFormatter formatter = new JodaDateFormatter(dateFormatterPattern, parser, parser);
DateFormatter formatter = dateFormatter.withZone(timeZone);
DateMathParser dateMathParser = formatter.toDateMathParser();
long millis = dateMathParser.parse(mathExpression, context::getStartTime, false,
DateUtils.dateTimeZoneToZoneId(timeZone));
long millis = dateMathParser.parse(mathExpression, context::getStartTime, false, timeZone);


String time = formatter.formatMillis(millis);
beforePlaceHolderSb.append(time);
inPlaceHolderSb = new StringBuilder();
Expand Down Expand Up @@ -1007,18 +1006,4 @@ String resolveExpression(String expression, final Context context) {
return beforePlaceHolderSb.toString();
}
}

/**
* Returns <code>true</code> iff the given expression resolves to the given index name otherwise <code>false</code>
*/
public final boolean matchesIndex(String indexName, String expression, ClusterState state) {
final String[] concreteIndices = concreteIndexNames(state, IndicesOptions.lenientExpandOpen(), expression);
for (String index : concreteIndices) {
if (Regex.simpleMatch(index, indexName)) {
return true;
}
}
return indexName.equals(expression);
}

}
7 changes: 0 additions & 7 deletions server/src/main/java/org/elasticsearch/common/time/DateFormatters.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1680,12 +1680,6 @@ public static ZonedDateTime from(TemporalAccessor accessor, Locale locale, ZoneI
} else {
return Year.of(accessor.get(ChronoField.YEAR)).atDay(1).atStartOfDay(zoneId);
}
} else if (accessor.isSupported(ChronoField.YEAR)) {
if (accessor.isSupported(MONTH_OF_YEAR)) {
return getFirstOfMonth(accessor).atStartOfDay(zoneId);
} else {
return Year.of(accessor.get(ChronoField.YEAR)).atDay(1).atStartOfDay(zoneId);
}
} else if (accessor.isSupported(MONTH_OF_YEAR)) {
// missing year, falling back to the epoch and then filling
return getLocaldate(accessor, locale).atStartOfDay(zoneId);
Expand Down Expand Up @@ -1713,7 +1707,6 @@ private static LocalDate localDateFromWeekBasedDate(TemporalAccessor accessor, L
}
}


private static LocalDate getLocaldate(TemporalAccessor accessor, Locale locale) {
if (accessor.isSupported(WeekFields.of(locale).weekBasedYear())) {
return localDateFromWeekBasedDate(accessor, locale);
Expand Down

0 comments on commit e49df55

Please sign in to comment.