diff --git a/core/src/main/java/org/apache/lucene/queryparser/classic/MapperQueryParser.java b/core/src/main/java/org/apache/lucene/queryparser/classic/MapperQueryParser.java index bbb27aa11db4b..ac9770f2bc8b1 100644 --- a/core/src/main/java/org/apache/lucene/queryparser/classic/MapperQueryParser.java +++ b/core/src/main/java/org/apache/lucene/queryparser/classic/MapperQueryParser.java @@ -24,6 +24,7 @@ import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; import org.apache.lucene.index.Term; +import org.apache.lucene.queryparser.analyzing.AnalyzingQueryParser; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.BoostQuery; @@ -34,6 +35,7 @@ import org.apache.lucene.search.PhraseQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.SynonymQuery; +import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.automaton.RegExp; import org.elasticsearch.common.lucene.search.Queries; @@ -42,6 +44,7 @@ import org.elasticsearch.index.mapper.LegacyDateFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.mapper.StringFieldType; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.query.support.QueryParsers; @@ -63,7 +66,7 @@ * Also breaks fields with [type].[name] into a boolean query that must include the type * as well as the query on the name. */ -public class MapperQueryParser extends QueryParser { +public class MapperQueryParser extends AnalyzingQueryParser { public static final Map FIELD_QUERY_EXTENSIONS; @@ -99,11 +102,10 @@ public void reset(QueryParserSettings settings) { setAutoGeneratePhraseQueries(settings.autoGeneratePhraseQueries()); setMaxDeterminizedStates(settings.maxDeterminizedStates()); setAllowLeadingWildcard(settings.allowLeadingWildcard()); - setLowercaseExpandedTerms(settings.lowercaseExpandedTerms()); + setLowercaseExpandedTerms(false); setPhraseSlop(settings.phraseSlop()); setDefaultOperator(settings.defaultOperator()); setFuzzyPrefixLength(settings.fuzzyPrefixLength()); - setLocale(settings.locale()); setSplitOnWhitespace(settings.splitOnWhitespace()); } @@ -330,21 +332,20 @@ private Query getRangeQuerySingle(String field, String part1, String part2, boolean startInclusive, boolean endInclusive, QueryShardContext context) { currentFieldType = context.fieldMapper(field); if (currentFieldType != null) { - if (lowercaseExpandedTerms && currentFieldType.tokenized()) { - part1 = part1 == null ? null : part1.toLowerCase(locale); - part2 = part2 == null ? null : part2.toLowerCase(locale); - } - try { + BytesRef part1Binary = part1 == null ? null : getAnalyzer().normalize(field, part1); + BytesRef part2Binary = part2 == null ? null : getAnalyzer().normalize(field, part2); Query rangeQuery; if (currentFieldType instanceof LegacyDateFieldMapper.DateFieldType && settings.timeZone() != null) { LegacyDateFieldMapper.DateFieldType dateFieldType = (LegacyDateFieldMapper.DateFieldType) this.currentFieldType; - rangeQuery = dateFieldType.rangeQuery(part1, part2, startInclusive, endInclusive, settings.timeZone(), null, context); + rangeQuery = dateFieldType.rangeQuery(part1Binary, part2Binary, + startInclusive, endInclusive, settings.timeZone(), null, context); } else if (currentFieldType instanceof DateFieldMapper.DateFieldType && settings.timeZone() != null) { DateFieldMapper.DateFieldType dateFieldType = (DateFieldMapper.DateFieldType) this.currentFieldType; - rangeQuery = dateFieldType.rangeQuery(part1, part2, startInclusive, endInclusive, settings.timeZone(), null, context); + rangeQuery = dateFieldType.rangeQuery(part1Binary, part2Binary, + startInclusive, endInclusive, settings.timeZone(), null, context); } else { - rangeQuery = currentFieldType.rangeQuery(part1, part2, startInclusive, endInclusive, context); + rangeQuery = currentFieldType.rangeQuery(part1Binary, part2Binary, startInclusive, endInclusive, context); } return rangeQuery; } catch (RuntimeException e) { @@ -358,9 +359,6 @@ private Query getRangeQuerySingle(String field, String part1, String part2, } protected Query getFuzzyQuery(String field, String termStr, String minSimilarity) throws ParseException { - if (lowercaseExpandedTerms) { - termStr = termStr.toLowerCase(locale); - } Collection fields = extractMultiFields(field); if (fields != null) { if (fields.size() == 1) { @@ -399,8 +397,9 @@ private Query getFuzzyQuerySingle(String field, String termStr, String minSimila currentFieldType = context.fieldMapper(field); if (currentFieldType != null) { try { - return currentFieldType.fuzzyQuery(termStr, Fuzziness.build(minSimilarity), - fuzzyPrefixLength, settings.fuzzyMaxExpansions(), FuzzyQuery.defaultTranspositions); + BytesRef term = termStr == null ? null : getAnalyzer().normalize(field, termStr); + return currentFieldType.fuzzyQuery(term, Fuzziness.build(minSimilarity), + getFuzzyPrefixLength(), settings.fuzzyMaxExpansions(), FuzzyQuery.defaultTranspositions); } catch (RuntimeException e) { if (settings.lenient()) { return null; @@ -423,9 +422,6 @@ protected Query newFuzzyQuery(Term term, float minimumSimilarity, int prefixLeng @Override protected Query getPrefixQuery(String field, String termStr) throws ParseException { - if (lowercaseExpandedTerms) { - termStr = termStr.toLowerCase(locale); - } Collection fields = extractMultiFields(field); if (fields != null) { if (fields.size() == 1) { @@ -471,8 +467,8 @@ private Query getPrefixQuerySingle(String field, String termStr) throws ParseExc setAnalyzer(context.getSearchAnalyzer(currentFieldType)); } Query query = null; - if (currentFieldType.tokenized() == false) { - query = currentFieldType.prefixQuery(termStr, multiTermRewriteMethod, context); + if (currentFieldType instanceof StringFieldType == false) { + query = currentFieldType.prefixQuery(termStr, getMultiTermRewriteMethod(), context); } if (query == null) { query = getPossiblyAnalyzedPrefixQuery(currentFieldType.name(), termStr); @@ -590,9 +586,6 @@ protected Query getWildcardQuery(String field, String termStr) throws ParseExcep return FIELD_QUERY_EXTENSIONS.get(ExistsFieldQueryExtension.NAME).query(context, actualField); } } - if (lowercaseExpandedTerms) { - termStr = termStr.toLowerCase(locale); - } Collection fields = extractMultiFields(field); if (fields != null) { if (fields.size() == 1) { @@ -639,9 +632,8 @@ private Query getWildcardQuerySingle(String field, String termStr) throws ParseE setAnalyzer(context.getSearchAnalyzer(currentFieldType)); } indexedNameField = currentFieldType.name(); - return getPossiblyAnalyzedWildcardQuery(indexedNameField, termStr); } - return getPossiblyAnalyzedWildcardQuery(indexedNameField, termStr); + return super.getWildcardQuery(indexedNameField, termStr); } catch (RuntimeException e) { if (settings.lenient()) { return null; @@ -652,75 +644,8 @@ private Query getWildcardQuerySingle(String field, String termStr) throws ParseE } } - private Query getPossiblyAnalyzedWildcardQuery(String field, String termStr) throws ParseException { - if (!settings.analyzeWildcard()) { - return super.getWildcardQuery(field, termStr); - } - boolean isWithinToken = (!termStr.startsWith("?") && !termStr.startsWith("*")); - StringBuilder aggStr = new StringBuilder(); - StringBuilder tmp = new StringBuilder(); - for (int i = 0; i < termStr.length(); i++) { - char c = termStr.charAt(i); - if (c == '?' || c == '*') { - if (isWithinToken) { - try (TokenStream source = getAnalyzer().tokenStream(field, tmp.toString())) { - source.reset(); - CharTermAttribute termAtt = source.addAttribute(CharTermAttribute.class); - if (source.incrementToken()) { - String term = termAtt.toString(); - if (term.length() == 0) { - // no tokens, just use what we have now - aggStr.append(tmp); - } else { - aggStr.append(term); - } - } else { - // no tokens, just use what we have now - aggStr.append(tmp); - } - } catch (IOException e) { - aggStr.append(tmp); - } - tmp.setLength(0); - } - isWithinToken = false; - aggStr.append(c); - } else { - tmp.append(c); - isWithinToken = true; - } - } - if (isWithinToken) { - try { - try (TokenStream source = getAnalyzer().tokenStream(field, tmp.toString())) { - source.reset(); - CharTermAttribute termAtt = source.addAttribute(CharTermAttribute.class); - if (source.incrementToken()) { - String term = termAtt.toString(); - if (term.length() == 0) { - // no tokens, just use what we have now - aggStr.append(tmp); - } else { - aggStr.append(term); - } - } else { - // no tokens, just use what we have now - aggStr.append(tmp); - } - } - } catch (IOException e) { - aggStr.append(tmp); - } - } - - return super.getWildcardQuery(field, aggStr.toString()); - } - @Override protected Query getRegexpQuery(String field, String termStr) throws ParseException { - if (lowercaseExpandedTerms) { - termStr = termStr.toLowerCase(locale); - } Collection fields = extractMultiFields(field); if (fields != null) { if (fields.size() == 1) { @@ -768,7 +693,7 @@ private Query getRegexpQuerySingle(String field, String termStr) throws ParseExc Query query = null; if (currentFieldType.tokenized() == false) { query = currentFieldType.regexpQuery(termStr, RegExp.ALL, - maxDeterminizedStates, multiTermRewriteMethod, context); + getMaxDeterminizedStates(), getMultiTermRewriteMethod(), context); } if (query == null) { query = super.getRegexpQuery(field, termStr); diff --git a/core/src/main/java/org/apache/lucene/queryparser/classic/QueryParserSettings.java b/core/src/main/java/org/apache/lucene/queryparser/classic/QueryParserSettings.java index a03c84d89803a..295c1ace4f637 100644 --- a/core/src/main/java/org/apache/lucene/queryparser/classic/QueryParserSettings.java +++ b/core/src/main/java/org/apache/lucene/queryparser/classic/QueryParserSettings.java @@ -24,7 +24,6 @@ import org.elasticsearch.common.unit.Fuzziness; import org.joda.time.DateTimeZone; -import java.util.Locale; import java.util.Map; /** @@ -53,12 +52,8 @@ public class QueryParserSettings { private boolean analyzeWildcard; - private boolean lowercaseExpandedTerms; - private boolean enablePositionIncrements; - private Locale locale; - private Fuzziness fuzziness; private int fuzzyPrefixLength; private int fuzzyMaxExpansions; @@ -137,14 +132,6 @@ public void allowLeadingWildcard(boolean allowLeadingWildcard) { this.allowLeadingWildcard = allowLeadingWildcard; } - public boolean lowercaseExpandedTerms() { - return lowercaseExpandedTerms; - } - - public void lowercaseExpandedTerms(boolean lowercaseExpandedTerms) { - this.lowercaseExpandedTerms = lowercaseExpandedTerms; - } - public boolean enablePositionIncrements() { return enablePositionIncrements; } @@ -269,14 +256,6 @@ public void useDisMax(boolean useDisMax) { this.useDisMax = useDisMax; } - public void locale(Locale locale) { - this.locale = locale; - } - - public Locale locale() { - return this.locale; - } - public void timeZone(DateTimeZone timeZone) { this.timeZone = timeZone; } diff --git a/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java index 3e5262f12fa8d..799c1baedbf1c 100755 --- a/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -54,7 +54,6 @@ import java.util.Map; import java.util.Set; import java.util.function.Function; -import java.util.function.LongSupplier; import java.util.function.Supplier; import java.util.stream.Collectors; diff --git a/core/src/main/java/org/elasticsearch/index/query/QueryStringQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/QueryStringQueryBuilder.java index d88484c98a62e..2867169ecbebf 100644 --- a/core/src/main/java/org/elasticsearch/index/query/QueryStringQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/QueryStringQueryBuilder.java @@ -64,7 +64,6 @@ public class QueryStringQueryBuilder extends AbstractQueryBuildertrue. - */ - public QueryStringQueryBuilder lowercaseExpandedTerms(boolean lowercaseExpandedTerms) { - this.lowercaseExpandedTerms = lowercaseExpandedTerms; - return this; - } - - public boolean lowercaseExpandedTerms() { - return this.lowercaseExpandedTerms; - } - /** * Set to true to enable position increments in result query. Defaults to * true. @@ -488,6 +478,11 @@ public int phraseSlop() { return phraseSlop; } + public QueryStringQueryBuilder rewrite(String rewrite) { + this.rewrite = rewrite; + return this; + } + /** * Set to true to enable analysis on wildcard and prefix queries. */ @@ -500,11 +495,6 @@ public Boolean analyzeWildcard() { return this.analyzeWildcard; } - public QueryStringQueryBuilder rewrite(String rewrite) { - this.rewrite = rewrite; - return this; - } - public String rewrite() { return this.rewrite; } @@ -543,15 +533,6 @@ public Boolean lenient() { return this.lenient; } - public QueryStringQueryBuilder locale(Locale locale) { - this.locale = locale == null ? DEFAULT_LOCALE : locale; - return this; - } - - public Locale locale() { - return this.locale; - } - /** * In case of date field, we can adjust the from/to fields using a timezone */ @@ -625,7 +606,6 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep if (this.allowLeadingWildcard != null) { builder.field(ALLOW_LEADING_WILDCARD_FIELD.getPreferredName(), this.allowLeadingWildcard); } - builder.field(LOWERCASE_EXPANDED_TERMS_FIELD.getPreferredName(), this.lowercaseExpandedTerms); builder.field(ENABLE_POSITION_INCREMENTS_FIELD.getPreferredName(), this.enablePositionIncrements); this.fuzziness.toXContent(builder, params); builder.field(FUZZY_PREFIX_LENGTH_FIELD.getPreferredName(), this.fuzzyPrefixLength); @@ -649,7 +629,6 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep if (this.lenient != null) { builder.field(LENIENT_FIELD.getPreferredName(), this.lenient); } - builder.field(LOCALE_FIELD.getPreferredName(), this.locale.toLanguageTag()); if (this.timeZone != null) { builder.field(TIME_ZONE_FIELD.getPreferredName(), this.timeZone.getID()); } @@ -671,7 +650,6 @@ public static Optional fromXContent(QueryParseContext p float boost = AbstractQueryBuilder.DEFAULT_BOOST; boolean autoGeneratePhraseQueries = QueryStringQueryBuilder.DEFAULT_AUTO_GENERATE_PHRASE_QUERIES; int maxDeterminizedStates = QueryStringQueryBuilder.DEFAULT_MAX_DETERMINED_STATES; - boolean lowercaseExpandedTerms = QueryStringQueryBuilder.DEFAULT_LOWERCASE_EXPANDED_TERMS; boolean enablePositionIncrements = QueryStringQueryBuilder.DEFAULT_ENABLE_POSITION_INCREMENTS; boolean escape = QueryStringQueryBuilder.DEFAULT_ESCAPE; boolean useDisMax = QueryStringQueryBuilder.DEFAULT_USE_DIS_MAX; @@ -686,7 +664,6 @@ public static Optional fromXContent(QueryParseContext p Boolean lenient = null; Operator defaultOperator = QueryStringQueryBuilder.DEFAULT_OPERATOR; String timeZone = null; - Locale locale = QueryStringQueryBuilder.DEFAULT_LOCALE; Fuzziness fuzziness = QueryStringQueryBuilder.DEFAULT_FUZZINESS; String fuzzyRewrite = null; String rewrite = null; @@ -737,7 +714,7 @@ public static Optional fromXContent(QueryParseContext p } else if (parseContext.getParseFieldMatcher().match(currentFieldName, MAX_DETERMINED_STATES_FIELD)) { maxDeterminizedStates = parser.intValue(); } else if (parseContext.getParseFieldMatcher().match(currentFieldName, LOWERCASE_EXPANDED_TERMS_FIELD)) { - lowercaseExpandedTerms = parser.booleanValue(); + // ignore, deprecated setting } else if (parseContext.getParseFieldMatcher().match(currentFieldName, ENABLE_POSITION_INCREMENTS_FIELD)) { enablePositionIncrements = parser.booleanValue(); } else if (parseContext.getParseFieldMatcher().match(currentFieldName, ESCAPE_FIELD)) { @@ -769,8 +746,7 @@ public static Optional fromXContent(QueryParseContext p } else if (parseContext.getParseFieldMatcher().match(currentFieldName, LENIENT_FIELD)) { lenient = parser.booleanValue(); } else if (parseContext.getParseFieldMatcher().match(currentFieldName, LOCALE_FIELD)) { - String localeStr = parser.text(); - locale = Locale.forLanguageTag(localeStr); + // ignore, deprecated setting } else if (parseContext.getParseFieldMatcher().match(currentFieldName, TIME_ZONE_FIELD)) { try { timeZone = parser.text(); @@ -804,7 +780,6 @@ public static Optional fromXContent(QueryParseContext p queryStringQuery.allowLeadingWildcard(allowLeadingWildcard); queryStringQuery.autoGeneratePhraseQueries(autoGeneratePhraseQueries); queryStringQuery.maxDeterminizedStates(maxDeterminizedStates); - queryStringQuery.lowercaseExpandedTerms(lowercaseExpandedTerms); queryStringQuery.enablePositionIncrements(enablePositionIncrements); queryStringQuery.escape(escape); queryStringQuery.useDisMax(useDisMax); @@ -820,7 +795,6 @@ public static Optional fromXContent(QueryParseContext p queryStringQuery.quoteFieldSuffix(quoteFieldSuffix); queryStringQuery.lenient(lenient); queryStringQuery.timeZone(timeZone); - queryStringQuery.locale(locale); queryStringQuery.boost(boost); queryStringQuery.queryName(queryName); queryStringQuery.splitOnWhitespace(splitOnWhitespace); @@ -843,10 +817,8 @@ protected boolean doEquals(QueryStringQueryBuilder other) { Objects.equals(quoteFieldSuffix, other.quoteFieldSuffix) && Objects.equals(autoGeneratePhraseQueries, other.autoGeneratePhraseQueries) && Objects.equals(allowLeadingWildcard, other.allowLeadingWildcard) && - Objects.equals(lowercaseExpandedTerms, other.lowercaseExpandedTerms) && Objects.equals(enablePositionIncrements, other.enablePositionIncrements) && Objects.equals(analyzeWildcard, other.analyzeWildcard) && - Objects.equals(locale.toLanguageTag(), other.locale.toLanguageTag()) && Objects.equals(fuzziness, other.fuzziness) && Objects.equals(fuzzyPrefixLength, other.fuzzyPrefixLength) && Objects.equals(fuzzyMaxExpansions, other.fuzzyMaxExpansions) && @@ -867,8 +839,8 @@ protected boolean doEquals(QueryStringQueryBuilder other) { @Override protected int doHashCode() { return Objects.hash(queryString, defaultField, fieldsAndWeights, defaultOperator, analyzer, quoteAnalyzer, - quoteFieldSuffix, autoGeneratePhraseQueries, allowLeadingWildcard, lowercaseExpandedTerms, - enablePositionIncrements, analyzeWildcard, locale.toLanguageTag(), fuzziness, fuzzyPrefixLength, + quoteFieldSuffix, autoGeneratePhraseQueries, allowLeadingWildcard, analyzeWildcard, + enablePositionIncrements, fuzziness, fuzzyPrefixLength, fuzzyMaxExpansions, fuzzyRewrite, phraseSlop, useDisMax, tieBreaker, rewrite, minimumShouldMatch, lenient, timeZone == null ? 0 : timeZone.getID(), escape, maxDeterminizedStates, splitOnWhitespace); } @@ -924,9 +896,7 @@ protected Query doToQuery(QueryShardContext context) throws IOException { qpSettings.autoGeneratePhraseQueries(autoGeneratePhraseQueries); qpSettings.allowLeadingWildcard(allowLeadingWildcard == null ? context.queryStringAllowLeadingWildcard() : allowLeadingWildcard); qpSettings.analyzeWildcard(analyzeWildcard == null ? context.queryStringAnalyzeWildcard() : analyzeWildcard); - qpSettings.lowercaseExpandedTerms(lowercaseExpandedTerms); qpSettings.enablePositionIncrements(enablePositionIncrements); - qpSettings.locale(locale); qpSettings.fuzziness(fuzziness); qpSettings.fuzzyPrefixLength(fuzzyPrefixLength); qpSettings.fuzzyMaxExpansions(fuzzyMaxExpansions); diff --git a/core/src/main/java/org/elasticsearch/index/query/SimpleQueryParser.java b/core/src/main/java/org/elasticsearch/index/query/SimpleQueryParser.java index e763b78c7f805..67d0d445113c6 100644 --- a/core/src/main/java/org/elasticsearch/index/query/SimpleQueryParser.java +++ b/core/src/main/java/org/elasticsearch/index/query/SimpleQueryParser.java @@ -30,10 +30,10 @@ import org.apache.lucene.search.PrefixQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.SynonymQuery; +import org.apache.lucene.util.BytesRef; import org.elasticsearch.index.mapper.MappedFieldType; import java.io.IOException; -import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.List; @@ -98,14 +98,13 @@ public Query newDefaultQuery(String text) { */ @Override public Query newFuzzyQuery(String text, int fuzziness) { - if (settings.lowercaseExpandedTerms()) { - text = text.toLowerCase(settings.locale()); - } BooleanQuery.Builder bq = new BooleanQuery.Builder(); bq.setDisableCoord(true); for (Map.Entry entry : weights.entrySet()) { + final String fieldName = entry.getKey(); try { - Query query = new FuzzyQuery(new Term(entry.getKey(), text), fuzziness); + final BytesRef term = getAnalyzer().normalize(fieldName, text); + Query query = new FuzzyQuery(new Term(fieldName, term), fuzziness); bq.add(wrapWithBoost(query, entry.getValue()), BooleanClause.Occur.SHOULD); } catch (RuntimeException e) { rethrowUnlessLenient(e); @@ -146,20 +145,19 @@ public Query newPhraseQuery(String text, int slop) { */ @Override public Query newPrefixQuery(String text) { - if (settings.lowercaseExpandedTerms()) { - text = text.toLowerCase(settings.locale()); - } BooleanQuery.Builder bq = new BooleanQuery.Builder(); bq.setDisableCoord(true); for (Map.Entry entry : weights.entrySet()) { + final String fieldName = entry.getKey(); try { if (settings.analyzeWildcard()) { - Query analyzedQuery = newPossiblyAnalyzedQuery(entry.getKey(), text); + Query analyzedQuery = newPossiblyAnalyzedQuery(fieldName, text); if (analyzedQuery != null) { bq.add(wrapWithBoost(analyzedQuery, entry.getValue()), BooleanClause.Occur.SHOULD); } } else { - Query query = new PrefixQuery(new Term(entry.getKey(), text)); + Term term = new Term(fieldName, getAnalyzer().normalize(fieldName, text)); + Query query = new PrefixQuery(term); bq.add(wrapWithBoost(query, entry.getValue()), BooleanClause.Occur.SHOULD); } } catch (RuntimeException e) { @@ -182,11 +180,11 @@ private static Query wrapWithBoost(Query query, float boost) { * of {@code TermQuery}s and {@code PrefixQuery}s */ private Query newPossiblyAnalyzedQuery(String field, String termStr) { - List> tlist = new ArrayList<> (); + List> tlist = new ArrayList<> (); // get Analyzer from superclass and tokenize the term try (TokenStream source = getAnalyzer().tokenStream(field, termStr)) { source.reset(); - List currentPos = new ArrayList<>(); + List currentPos = new ArrayList<>(); CharTermAttribute termAtt = source.addAttribute(CharTermAttribute.class); PositionIncrementAttribute posAtt = source.addAttribute(PositionIncrementAttribute.class); @@ -197,7 +195,8 @@ private Query newPossiblyAnalyzedQuery(String field, String termStr) { tlist.add(currentPos); currentPos = new ArrayList<>(); } - currentPos.add(termAtt.toString()); + final BytesRef term = getAnalyzer().normalize(field, termAtt.toString()); + currentPos.add(term); hasMoreTokens = source.incrementToken(); } if (currentPos.isEmpty() == false) { @@ -223,7 +222,7 @@ private Query newPossiblyAnalyzedQuery(String field, String termStr) { // build a boolean query with prefix on the last position only. BooleanQuery.Builder builder = new BooleanQuery.Builder(); for (int pos = 0; pos < tlist.size(); pos++) { - List plist = tlist.get(pos); + List plist = tlist.get(pos); boolean isLastPos = (pos == tlist.size()-1); Query posQuery; if (plist.size() == 1) { @@ -241,7 +240,7 @@ private Query newPossiblyAnalyzedQuery(String field, String termStr) { posQuery = new SynonymQuery(terms); } else { BooleanQuery.Builder innerBuilder = new BooleanQuery.Builder(); - for (String token : plist) { + for (BytesRef token : plist) { innerBuilder.add(new BooleanClause(new PrefixQuery(new Term(field, token)), BooleanClause.Occur.SHOULD)); } @@ -257,10 +256,6 @@ private Query newPossiblyAnalyzedQuery(String field, String termStr) { * their default values */ static class Settings { - /** Locale to use for parsing. */ - private Locale locale = SimpleQueryStringBuilder.DEFAULT_LOCALE; - /** Specifies whether parsed terms should be lowercased. */ - private boolean lowercaseExpandedTerms = SimpleQueryStringBuilder.DEFAULT_LOWERCASE_EXPANDED_TERMS; /** Specifies whether lenient query parsing should be used. */ private boolean lenient = SimpleQueryStringBuilder.DEFAULT_LENIENT; /** Specifies whether wildcards should be analyzed. */ @@ -275,29 +270,6 @@ static class Settings { public Settings() { } - /** Specifies the locale to use for parsing, Locale.ROOT by default. */ - public void locale(Locale locale) { - this.locale = (locale != null) ? locale : SimpleQueryStringBuilder.DEFAULT_LOCALE; - } - - /** Returns the locale to use for parsing. */ - public Locale locale() { - return this.locale; - } - - /** - * Specifies whether to lowercase parse terms, defaults to true if - * unset. - */ - public void lowercaseExpandedTerms(boolean lowercaseExpandedTerms) { - this.lowercaseExpandedTerms = lowercaseExpandedTerms; - } - - /** Returns whether to lowercase parse terms. */ - public boolean lowercaseExpandedTerms() { - return this.lowercaseExpandedTerms; - } - /** Specifies whether to use lenient parsing, defaults to false. */ public void lenient(boolean lenient) { this.lenient = lenient; @@ -335,10 +307,7 @@ public String quoteFieldSuffix() { @Override public int hashCode() { - // checking the return value of toLanguageTag() for locales only. - // For further reasoning see - // https://issues.apache.org/jira/browse/LUCENE-4021 - return Objects.hash(locale.toLanguageTag(), lowercaseExpandedTerms, lenient, analyzeWildcard, quoteFieldSuffix); + return Objects.hash(lenient, analyzeWildcard, quoteFieldSuffix); } @Override @@ -350,15 +319,8 @@ public boolean equals(Object obj) { return false; } Settings other = (Settings) obj; - - // checking the return value of toLanguageTag() for locales only. - // For further reasoning see - // https://issues.apache.org/jira/browse/LUCENE-4021 - return (Objects.equals(locale.toLanguageTag(), other.locale.toLanguageTag()) - && Objects.equals(lowercaseExpandedTerms, other.lowercaseExpandedTerms) - && Objects.equals(lenient, other.lenient) - && Objects.equals(analyzeWildcard, other.analyzeWildcard) - && Objects.equals(quoteFieldSuffix, other.quoteFieldSuffix)); + return Objects.equals(lenient, other.lenient) && Objects.equals(analyzeWildcard, other.analyzeWildcard) + && Objects.equals(quoteFieldSuffix, other.quoteFieldSuffix); } } } diff --git a/core/src/main/java/org/elasticsearch/index/query/SimpleQueryStringBuilder.java b/core/src/main/java/org/elasticsearch/index/query/SimpleQueryStringBuilder.java index 15c0e0f4f15d6..fd29707506744 100644 --- a/core/src/main/java/org/elasticsearch/index/query/SimpleQueryStringBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/SimpleQueryStringBuilder.java @@ -79,10 +79,6 @@ * > online documentation. */ public class SimpleQueryStringBuilder extends AbstractQueryBuilder { - /** Default locale used for parsing.*/ - public static final Locale DEFAULT_LOCALE = Locale.ROOT; - /** Default for lowercasing parsed terms.*/ - public static final boolean DEFAULT_LOWERCASE_EXPANDED_TERMS = true; /** Default for using lenient query parsing.*/ public static final boolean DEFAULT_LENIENT = false; /** Default for wildcard analysis.*/ @@ -100,8 +96,10 @@ public class SimpleQueryStringBuilder extends AbstractQueryBuilder fromXContent(QueryParseContext String analyzerName = null; int flags = SimpleQueryStringFlag.ALL.value(); boolean lenient = SimpleQueryStringBuilder.DEFAULT_LENIENT; - boolean lowercaseExpandedTerms = SimpleQueryStringBuilder.DEFAULT_LOWERCASE_EXPANDED_TERMS; boolean analyzeWildcard = SimpleQueryStringBuilder.DEFAULT_ANALYZE_WILDCARD; - Locale locale = null; String quoteFieldSuffix = null; XContentParser.Token token; @@ -512,10 +489,9 @@ public static Optional fromXContent(QueryParseContext } } } else if (parseContext.getParseFieldMatcher().match(currentFieldName, LOCALE_FIELD)) { - String localeStr = parser.text(); - locale = Locale.forLanguageTag(localeStr); + // ignore, deprecated setting } else if (parseContext.getParseFieldMatcher().match(currentFieldName, LOWERCASE_EXPANDED_TERMS_FIELD)) { - lowercaseExpandedTerms = parser.booleanValue(); + // ignore, deprecated setting } else if (parseContext.getParseFieldMatcher().match(currentFieldName, LENIENT_FIELD)) { lenient = parser.booleanValue(); } else if (parseContext.getParseFieldMatcher().match(currentFieldName, ANALYZE_WILDCARD_FIELD)) { @@ -543,7 +519,7 @@ public static Optional fromXContent(QueryParseContext SimpleQueryStringBuilder qb = new SimpleQueryStringBuilder(queryBody); qb.boost(boost).fields(fieldsAndWeights).analyzer(analyzerName).queryName(queryName).minimumShouldMatch(minimumShouldMatch); - qb.flags(flags).defaultOperator(defaultOperator).locale(locale).lowercaseExpandedTerms(lowercaseExpandedTerms); + qb.flags(flags).defaultOperator(defaultOperator); qb.lenient(lenient).analyzeWildcard(analyzeWildcard).boost(boost).quoteFieldSuffix(quoteFieldSuffix); return Optional.of(qb); } diff --git a/core/src/main/java/org/elasticsearch/rest/action/RestActions.java b/core/src/main/java/org/elasticsearch/rest/action/RestActions.java index c5a3799a88d94..d017dfaf87476 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/RestActions.java +++ b/core/src/main/java/org/elasticsearch/rest/action/RestActions.java @@ -192,7 +192,6 @@ public static QueryBuilder urlParamsToQueryBuilder(RestRequest request) { queryBuilder.defaultField(request.param("df")); queryBuilder.analyzer(request.param("analyzer")); queryBuilder.analyzeWildcard(request.paramAsBoolean("analyze_wildcard", false)); - queryBuilder.lowercaseExpandedTerms(request.paramAsBoolean("lowercase_expanded_terms", true)); queryBuilder.lenient(request.paramAsBoolean("lenient", null)); String defaultOperator = request.param("default_operator"); if (defaultOperator != null) { diff --git a/core/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java index 73b3711701dc8..d89cb6702e04c 100644 --- a/core/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java @@ -36,6 +36,9 @@ import org.apache.lucene.search.RegexpQuery; import org.apache.lucene.search.SynonymQuery; import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.TermRangeQuery; +import org.apache.lucene.search.WildcardQuery; +import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.automaton.TooComplexToDeterminizeException; import org.elasticsearch.common.lucene.all.AllTermQuery; import org.elasticsearch.common.unit.Fuzziness; @@ -104,9 +107,6 @@ protected QueryStringQueryBuilder doCreateTestQueryBuilder() { if (randomBoolean()) { queryStringQueryBuilder.maxDeterminizedStates(randomIntBetween(1, 100)); } - if (randomBoolean()) { - queryStringQueryBuilder.lowercaseExpandedTerms(randomBoolean()); - } if (randomBoolean()) { queryStringQueryBuilder.autoGeneratePhraseQueries(randomBoolean()); } @@ -146,9 +146,6 @@ protected QueryStringQueryBuilder doCreateTestQueryBuilder() { if (randomBoolean()) { queryStringQueryBuilder.useDisMax(randomBoolean()); } - if (randomBoolean()) { - queryStringQueryBuilder.locale(randomLocale(random())); - } if (randomBoolean()) { queryStringQueryBuilder.timeZone(randomDateTimeZone().getID()); } @@ -669,13 +666,11 @@ public void testFromJson() throws IOException { " \"default_operator\" : \"or\",\n" + " \"auto_generate_phrase_queries\" : false,\n" + " \"max_determined_states\" : 10000,\n" + - " \"lowercase_expanded_terms\" : true,\n" + " \"enable_position_increment\" : true,\n" + " \"fuzziness\" : \"AUTO\",\n" + " \"fuzzy_prefix_length\" : 0,\n" + " \"fuzzy_max_expansions\" : 50,\n" + " \"phrase_slop\" : 0,\n" + - " \"locale\" : \"und\",\n" + " \"escape\" : false,\n" + " \"split_on_whitespace\" : true,\n" + " \"boost\" : 1.0\n" + @@ -689,4 +684,56 @@ public void testFromJson() throws IOException { assertEquals(json, "content", parsed.defaultField()); } + public void testExpandedTerms() throws Exception { + // Prefix + Query query = new QueryStringQueryBuilder("aBc*") + .field(STRING_FIELD_NAME) + .analyzer("whitespace") + .toQuery(createShardContext()); + assertEquals(new PrefixQuery(new Term(STRING_FIELD_NAME, "aBc")), query); + query = new QueryStringQueryBuilder("aBc*") + .field(STRING_FIELD_NAME) + .analyzer("standard") + .toQuery(createShardContext()); + assertEquals(new PrefixQuery(new Term(STRING_FIELD_NAME, "abc")), query); + + // Wildcard + query = new QueryStringQueryBuilder("aBc*D") + .field(STRING_FIELD_NAME) + .analyzer("whitespace") + .toQuery(createShardContext()); + assertEquals(new WildcardQuery(new Term(STRING_FIELD_NAME, "aBc*D")), query); + query = new QueryStringQueryBuilder("aBc*D") + .field(STRING_FIELD_NAME) + .analyzer("standard") + .toQuery(createShardContext()); + assertEquals(new WildcardQuery(new Term(STRING_FIELD_NAME, "abc*d")), query); + + // Fuzzy + query = new QueryStringQueryBuilder("aBc~1") + .field(STRING_FIELD_NAME) + .analyzer("whitespace") + .toQuery(createShardContext()); + FuzzyQuery fuzzyQuery = (FuzzyQuery) query; + assertEquals(new Term(STRING_FIELD_NAME, "aBc"), fuzzyQuery.getTerm()); + query = new QueryStringQueryBuilder("aBc~1") + .field(STRING_FIELD_NAME) + .analyzer("standard") + .toQuery(createShardContext()); + fuzzyQuery = (FuzzyQuery) query; + assertEquals(new Term(STRING_FIELD_NAME, "abc"), fuzzyQuery.getTerm()); + + // Range + query = new QueryStringQueryBuilder("[aBc TO BcD]") + .field(STRING_FIELD_NAME) + .analyzer("whitespace") + .toQuery(createShardContext()); + assertEquals(new TermRangeQuery(STRING_FIELD_NAME, new BytesRef("aBc"), new BytesRef("BcD"), true, true), query); + query = new QueryStringQueryBuilder("[aBc TO BcD]") + .field(STRING_FIELD_NAME) + .analyzer("standard") + .toQuery(createShardContext()); + assertEquals(new TermRangeQuery(STRING_FIELD_NAME, new BytesRef("abc"), new BytesRef("bcd"), true, true), query); + } + } diff --git a/core/src/test/java/org/elasticsearch/index/query/SimpleQueryStringBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/SimpleQueryStringBuilderTests.java index 77257cdce6ad4..a8a1de059e882 100644 --- a/core/src/test/java/org/elasticsearch/index/query/SimpleQueryStringBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/index/query/SimpleQueryStringBuilderTests.java @@ -19,14 +19,14 @@ package org.elasticsearch.index.query; -import com.carrotsearch.randomizedtesting.generators.RandomStrings; - import org.apache.lucene.index.Term; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.BoostQuery; +import org.apache.lucene.search.FuzzyQuery; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.MatchNoDocsQuery; +import org.apache.lucene.search.PrefixQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; import org.apache.lucene.util.TestUtil; @@ -59,12 +59,6 @@ protected SimpleQueryStringBuilder doCreateTestQueryBuilder() { if (randomBoolean()) { result.lenient(randomBoolean()); } - if (randomBoolean()) { - result.lowercaseExpandedTerms(randomBoolean()); - } - if (randomBoolean()) { - result.locale(randomLocale(random())); - } if (randomBoolean()) { result.minimumShouldMatch(randomMinimumShouldMatch()); } @@ -115,28 +109,11 @@ public void testDefaults() { assertEquals("Wrong default default operator.", Operator.OR, qb.defaultOperator()); assertEquals("Wrong default default operator field.", Operator.OR, SimpleQueryStringBuilder.DEFAULT_OPERATOR); - assertEquals("Wrong default default locale.", Locale.ROOT, qb.locale()); - assertEquals("Wrong default default locale field.", Locale.ROOT, SimpleQueryStringBuilder.DEFAULT_LOCALE); - assertEquals("Wrong default default analyze_wildcard.", false, qb.analyzeWildcard()); assertEquals("Wrong default default analyze_wildcard field.", false, SimpleQueryStringBuilder.DEFAULT_ANALYZE_WILDCARD); - assertEquals("Wrong default default lowercase_expanded_terms.", true, qb.lowercaseExpandedTerms()); - assertEquals("Wrong default default lowercase_expanded_terms field.", true, - SimpleQueryStringBuilder.DEFAULT_LOWERCASE_EXPANDED_TERMS); - assertEquals("Wrong default default lenient.", false, qb.lenient()); assertEquals("Wrong default default lenient field.", false, SimpleQueryStringBuilder.DEFAULT_LENIENT); - - assertEquals("Wrong default default locale.", Locale.ROOT, qb.locale()); - assertEquals("Wrong default default locale field.", Locale.ROOT, SimpleQueryStringBuilder.DEFAULT_LOCALE); - } - - public void testDefaultNullLocale() { - SimpleQueryStringBuilder qb = new SimpleQueryStringBuilder("The quick brown fox."); - qb.locale(null); - assertEquals("Setting locale to null should result in returning to default value.", SimpleQueryStringBuilder.DEFAULT_LOCALE, - qb.locale()); } public void testDefaultNullComplainFlags() { @@ -256,12 +233,10 @@ protected void doAssertLuceneQuery(SimpleQueryStringBuilder queryBuilder, Query } else if (queryBuilder.fields().size() > 1) { assertThat(query, instanceOf(BooleanQuery.class)); BooleanQuery boolQuery = (BooleanQuery) query; - if (queryBuilder.lowercaseExpandedTerms()) { - for (BooleanClause clause : boolQuery.clauses()) { - if (clause.getQuery() instanceof TermQuery) { - TermQuery inner = (TermQuery) clause.getQuery(); - assertThat(inner.getTerm().bytes().toString(), is(inner.getTerm().bytes().toString().toLowerCase(Locale.ROOT))); - } + for (BooleanClause clause : boolQuery.clauses()) { + if (clause.getQuery() instanceof TermQuery) { + TermQuery inner = (TermQuery) clause.getQuery(); + assertThat(inner.getTerm().bytes().toString(), is(inner.getTerm().bytes().toString().toLowerCase(Locale.ROOT))); } } assertThat(boolQuery.clauses().size(), equalTo(queryBuilder.fields().size())); @@ -336,10 +311,8 @@ public void testFromJson() throws IOException { " \"analyzer\" : \"snowball\",\n" + " \"flags\" : -1,\n" + " \"default_operator\" : \"and\",\n" + - " \"lowercase_expanded_terms\" : true,\n" + " \"lenient\" : false,\n" + " \"analyze_wildcard\" : false,\n" + - " \"locale\" : \"und\",\n" + " \"quote_field_suffix\" : \".quote\",\n" + " \"boost\" : 1.0\n" + " }\n" + @@ -397,4 +370,32 @@ public void testIndexMetaField() throws IOException { assertThat(query, instanceOf(MatchAllDocsQuery.class)); } } + + public void testExpandedTerms() throws Exception { + // Prefix + Query query = new SimpleQueryStringBuilder("aBc*") + .field(STRING_FIELD_NAME) + .analyzer("whitespace") + .toQuery(createShardContext()); + assertEquals(new PrefixQuery(new Term(STRING_FIELD_NAME, "aBc")), query); + query = new SimpleQueryStringBuilder("aBc*") + .field(STRING_FIELD_NAME) + .analyzer("standard") + .toQuery(createShardContext()); + assertEquals(new PrefixQuery(new Term(STRING_FIELD_NAME, "abc")), query); + + // Fuzzy + query = new SimpleQueryStringBuilder("aBc~1") + .field(STRING_FIELD_NAME) + .analyzer("whitespace") + .toQuery(createShardContext()); + FuzzyQuery expected = new FuzzyQuery(new Term(STRING_FIELD_NAME, "aBc"), 1); + assertEquals(expected, query); + query = new SimpleQueryStringBuilder("aBc~1") + .field(STRING_FIELD_NAME) + .analyzer("standard") + .toQuery(createShardContext()); + expected = new FuzzyQuery(new Term(STRING_FIELD_NAME, "abc"), 1); + assertEquals(expected, query); + } } diff --git a/core/src/test/java/org/elasticsearch/search/child/ChildQuerySearchIT.java b/core/src/test/java/org/elasticsearch/search/child/ChildQuerySearchIT.java index d190f76b3fcab..a3ecc66c030a9 100644 --- a/core/src/test/java/org/elasticsearch/search/child/ChildQuerySearchIT.java +++ b/core/src/test/java/org/elasticsearch/search/child/ChildQuerySearchIT.java @@ -1148,7 +1148,7 @@ public void testHasChildQueryOnlyReturnsSingleChildType() { hasChildQuery( "child_type_one", boolQuery().must( - queryStringQuery("name:William*").analyzeWildcard(true) + queryStringQuery("name:William*") ), ScoreMode.None) ), @@ -1165,7 +1165,7 @@ public void testHasChildQueryOnlyReturnsSingleChildType() { hasChildQuery( "child_type_two", boolQuery().must( - queryStringQuery("name:William*").analyzeWildcard(true) + queryStringQuery("name:William*") ), ScoreMode.None) ), diff --git a/core/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java b/core/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java index 6ee6f56c5360a..1cb9d6508aa1b 100644 --- a/core/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java +++ b/core/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java @@ -480,19 +480,19 @@ public void testQueryStringAnalyzedWildcard() throws Exception { client().prepareIndex("test", "type1", "1").setSource("field1", "value_1", "field2", "value_2").get(); refresh(); - SearchResponse searchResponse = client().prepareSearch().setQuery(queryStringQuery("value*").analyzeWildcard(true)).get(); + SearchResponse searchResponse = client().prepareSearch().setQuery(queryStringQuery("value*")).get(); assertHitCount(searchResponse, 1L); - searchResponse = client().prepareSearch().setQuery(queryStringQuery("*ue*").analyzeWildcard(true)).get(); + searchResponse = client().prepareSearch().setQuery(queryStringQuery("*ue*")).get(); assertHitCount(searchResponse, 1L); - searchResponse = client().prepareSearch().setQuery(queryStringQuery("*ue_1").analyzeWildcard(true)).get(); + searchResponse = client().prepareSearch().setQuery(queryStringQuery("*ue_1")).get(); assertHitCount(searchResponse, 1L); - searchResponse = client().prepareSearch().setQuery(queryStringQuery("val*e_1").analyzeWildcard(true)).get(); + searchResponse = client().prepareSearch().setQuery(queryStringQuery("val*e_1")).get(); assertHitCount(searchResponse, 1L); - searchResponse = client().prepareSearch().setQuery(queryStringQuery("v?l*e?1").analyzeWildcard(true)).get(); + searchResponse = client().prepareSearch().setQuery(queryStringQuery("v?l*e?1")).get(); assertHitCount(searchResponse, 1L); } @@ -502,18 +502,14 @@ public void testLowercaseExpandedTerms() { client().prepareIndex("test", "type1", "1").setSource("field1", "value_1", "field2", "value_2").get(); refresh(); - SearchResponse searchResponse = client().prepareSearch().setQuery(queryStringQuery("VALUE_3~1").lowercaseExpandedTerms(true)).get(); + SearchResponse searchResponse = client().prepareSearch().setQuery(queryStringQuery("VALUE_3~1")).get(); assertHitCount(searchResponse, 1L); - searchResponse = client().prepareSearch().setQuery(queryStringQuery("VALUE_3~1").lowercaseExpandedTerms(false)).get(); - assertHitCount(searchResponse, 0L); - searchResponse = client().prepareSearch().setQuery(queryStringQuery("ValUE_*").lowercaseExpandedTerms(true)).get(); + searchResponse = client().prepareSearch().setQuery(queryStringQuery("ValUE_*")).get(); assertHitCount(searchResponse, 1L); searchResponse = client().prepareSearch().setQuery(queryStringQuery("vAl*E_1")).get(); assertHitCount(searchResponse, 1L); searchResponse = client().prepareSearch().setQuery(queryStringQuery("[VALUE_1 TO VALUE_3]")).get(); assertHitCount(searchResponse, 1L); - searchResponse = client().prepareSearch().setQuery(queryStringQuery("[VALUE_1 TO VALUE_3]").lowercaseExpandedTerms(false)).get(); - assertHitCount(searchResponse, 0L); } // Issue #3540 @@ -532,11 +528,11 @@ public void testDateRangeInQueryString() { SearchResponse searchResponse = client().prepareSearch().setQuery(queryStringQuery("past:[now-2M/d TO now/d]")).get(); assertHitCount(searchResponse, 1L); - searchResponse = client().prepareSearch().setQuery(queryStringQuery("future:[now/d TO now+2M/d]").lowercaseExpandedTerms(false)).get(); + searchResponse = client().prepareSearch().setQuery(queryStringQuery("future:[now/d TO now+2M/d]")).get(); assertHitCount(searchResponse, 1L); SearchPhaseExecutionException e = expectThrows(SearchPhaseExecutionException.class, () -> client().prepareSearch() - .setQuery(queryStringQuery("future:[now/D TO now+2M/d]").lowercaseExpandedTerms(false)).get()); + .setQuery(queryStringQuery("future:[now/D TO now+2M/d]")).get()); assertThat(e.status(), equalTo(RestStatus.BAD_REQUEST)); assertThat(e.toString(), containsString("unit [D] not supported for date math")); } diff --git a/core/src/test/java/org/elasticsearch/search/query/SimpleQueryStringIT.java b/core/src/test/java/org/elasticsearch/search/query/SimpleQueryStringIT.java index 9502a8183152e..b98bc5d43cda8 100644 --- a/core/src/test/java/org/elasticsearch/search/query/SimpleQueryStringIT.java +++ b/core/src/test/java/org/elasticsearch/search/query/SimpleQueryStringIT.java @@ -30,12 +30,10 @@ import org.elasticsearch.test.ESIntegTestCase; import java.io.IOException; -import java.util.Locale; import java.util.concurrent.ExecutionException; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.index.query.QueryBuilders.boolQuery; -import static org.elasticsearch.index.query.QueryBuilders.queryStringQuery; import static org.elasticsearch.index.query.QueryBuilders.simpleQueryStringQuery; import static org.elasticsearch.index.query.QueryBuilders.termQuery; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; @@ -158,49 +156,6 @@ public void testSimpleQueryStringMinimumShouldMatch() throws Exception { assertSearchHits(searchResponse, "6", "7", "8"); } - public void testSimpleQueryStringLowercasing() { - createIndex("test"); - client().prepareIndex("test", "type1", "1").setSource("body", "Professional").get(); - refresh(); - - SearchResponse searchResponse = client().prepareSearch().setQuery(simpleQueryStringQuery("Professio*")).get(); - assertHitCount(searchResponse, 1L); - assertSearchHits(searchResponse, "1"); - - searchResponse = client().prepareSearch().setQuery( - simpleQueryStringQuery("Professio*").lowercaseExpandedTerms(false)).get(); - assertHitCount(searchResponse, 0L); - - searchResponse = client().prepareSearch().setQuery( - simpleQueryStringQuery("Professionan~1")).get(); - assertHitCount(searchResponse, 1L); - assertSearchHits(searchResponse, "1"); - - searchResponse = client().prepareSearch().setQuery( - simpleQueryStringQuery("Professionan~1").lowercaseExpandedTerms(false)).get(); - assertHitCount(searchResponse, 0L); - } - - public void testQueryStringLocale() { - createIndex("test"); - client().prepareIndex("test", "type1", "1").setSource("body", "bılly").get(); - refresh(); - - SearchResponse searchResponse = client().prepareSearch().setQuery(simpleQueryStringQuery("BILL*")).get(); - assertHitCount(searchResponse, 0L); - searchResponse = client().prepareSearch().setQuery(queryStringQuery("body:BILL*")).get(); - assertHitCount(searchResponse, 0L); - - searchResponse = client().prepareSearch().setQuery( - simpleQueryStringQuery("BILL*").locale(new Locale("tr", "TR"))).get(); - assertHitCount(searchResponse, 1L); - assertSearchHits(searchResponse, "1"); - searchResponse = client().prepareSearch().setQuery( - queryStringQuery("body:BILL*").locale(new Locale("tr", "TR"))).get(); - assertHitCount(searchResponse, 1L); - assertSearchHits(searchResponse, "1"); - } - public void testNestedFieldSimpleQueryString() throws IOException { assertAcked(prepareCreate("test") .addMapping("type1", jsonBuilder() @@ -342,7 +297,7 @@ public void testSimpleQueryStringAnalyzeWildcard() throws ExecutionException, In refresh(); SearchResponse searchResponse = client().prepareSearch() - .setQuery(simpleQueryStringQuery("Köln*").analyzeWildcard(true).field("location")).get(); + .setQuery(simpleQueryStringQuery("Köln*").field("location")).get(); assertNoFailures(searchResponse); assertHitCount(searchResponse, 1L); assertSearchHits(searchResponse, "1"); @@ -393,7 +348,7 @@ public void testEmptySimpleQueryStringWithAnalysis() throws Exception { refresh(); SearchResponse searchResponse = client().prepareSearch() - .setQuery(simpleQueryStringQuery("the*").analyzeWildcard(true).field("body")).get(); + .setQuery(simpleQueryStringQuery("the*").field("body")).get(); assertNoFailures(searchResponse); assertHitCount(searchResponse, 0L); } diff --git a/docs/reference/query-dsl/query-string-query.asciidoc b/docs/reference/query-dsl/query-string-query.asciidoc index b8ce68721a1e8..28d8fcffbe262 100644 --- a/docs/reference/query-dsl/query-string-query.asciidoc +++ b/docs/reference/query-dsl/query-string-query.asciidoc @@ -40,10 +40,6 @@ with default operator of `AND`, the same query is translated to |`allow_leading_wildcard` |When set, `*` or `?` are allowed as the first character. Defaults to `true`. -|`lowercase_expanded_terms` |Whether terms of wildcard, prefix, fuzzy, -and range queries are to be automatically lower-cased or not (since they -are not analyzed). Default it `true`. - |`enable_position_increments` |Set to `true` to enable position increments in result queries. Defaults to `true`. @@ -61,12 +57,12 @@ phrase matches are required. Default value is `0`. |`boost` |Sets the boost value of the query. Defaults to `1.0`. +|`auto_generate_phrase_queries` |Defaults to `false`. + |`analyze_wildcard` |By default, wildcards terms in a query string are not analyzed. By setting this value to `true`, a best effort will be made to analyze those as well. -|`auto_generate_phrase_queries` |Defaults to `false`. - |`max_determinized_states` |Limit on how many automaton states regexp queries are allowed to create. This protects against too-difficult (e.g. exponentially hard) regexps. Defaults to 10000. @@ -80,9 +76,6 @@ both>>. |`lenient` |If set to `true` will cause format based failures (like providing text to a numeric field) to be ignored. -|`locale` | Locale that should be used for string conversions. -Defaults to `ROOT`. - |`time_zone` | Time Zone to be applied to any range query related to dates. See also http://www.joda.org/joda-time/apidocs/org/joda/time/DateTimeZone.html[JODA timezone]. diff --git a/docs/reference/query-dsl/query-string-syntax.asciidoc b/docs/reference/query-dsl/query-string-syntax.asciidoc index 49a8b54c5062b..9c9009591968c 100644 --- a/docs/reference/query-dsl/query-string-syntax.asciidoc +++ b/docs/reference/query-dsl/query-string-syntax.asciidoc @@ -61,12 +61,15 @@ they match. Leading wildcards can be disabled by setting `allow_leading_wildcard` to `false`. ======= -Wildcarded terms are not analyzed by default -- they are lowercased -(`lowercase_expanded_terms` defaults to `true`) but no further analysis -is done, mainly because it is impossible to accurately analyze a word that -is missing some of its letters. However, by setting `analyze_wildcard` to -`true`, an attempt will be made to analyze wildcarded words before searching -the term list for matching terms. +Only parts of the analysis chain that operate at the character level are +applied. So for instance, if the analyzer performs both lowercasing and +stemming, only the lowercasing will be applied: it would be wrong to perform +stemming on a word that is missing some of its letters. + +By setting `analyze_wildcard` to true, queries that end with a `*` will be +analyzed and a boolean query will be built out of the different tokens, by +ensuring exact matches on the first N-1 tokens, and prefix match on the last +token. ===== Regular expressions diff --git a/docs/reference/query-dsl/simple-query-string-query.asciidoc b/docs/reference/query-dsl/simple-query-string-query.asciidoc index 59e3f727fe038..c6f70c314160e 100644 --- a/docs/reference/query-dsl/simple-query-string-query.asciidoc +++ b/docs/reference/query-dsl/simple-query-string-query.asciidoc @@ -44,18 +44,11 @@ creating composite queries. |`flags` |Flags specifying which features of the `simple_query_string` to enable. Defaults to `ALL`. -|`lowercase_expanded_terms` | Whether terms of prefix and fuzzy queries should -be automatically lower-cased or not (since they are not analyzed). Defaults to -`true`. - |`analyze_wildcard` | Whether terms of prefix queries should be automatically analyzed or not. If `true` a best effort will be made to analyze the prefix. However, some analyzers will be not able to provide a meaningful results based just on the prefix of a term. Defaults to `false`. -|`locale` | Locale that should be used for string conversions. -Defaults to `ROOT`. - |`lenient` | If set to `true` will cause format based failures (like providing text to a numeric field) to be ignored. diff --git a/docs/reference/search/count.asciidoc b/docs/reference/search/count.asciidoc index 859455e89b72c..d4117e4e96eb7 100644 --- a/docs/reference/search/count.asciidoc +++ b/docs/reference/search/count.asciidoc @@ -74,9 +74,6 @@ query. |`lenient` |If set to true will cause format based failures (like providing text to a numeric field) to be ignored. Defaults to false. -|`lowercase_expanded_terms` |Should terms be automatically lowercased or -not. Defaults to `true`. - |`analyze_wildcard` |Should wildcard and prefix queries be analyzed or not. Defaults to `false`. diff --git a/docs/reference/search/explain.asciidoc b/docs/reference/search/explain.asciidoc index 1b6560abab460..1291af702f546 100644 --- a/docs/reference/search/explain.asciidoc +++ b/docs/reference/search/explain.asciidoc @@ -136,10 +136,6 @@ This will yield the same result as the previous request. Should wildcard and prefix queries be analyzed or not. Defaults to false. -`lowercase_expanded_terms`:: - Should terms be automatically lowercased - or not. Defaults to true. - `lenient`:: If set to true will cause format based failures (like providing text to a numeric field) to be ignored. Defaults to false. diff --git a/docs/reference/search/uri-request.asciidoc b/docs/reference/search/uri-request.asciidoc index 95ce6a8ff6a29..6670b9f31d574 100644 --- a/docs/reference/search/uri-request.asciidoc +++ b/docs/reference/search/uri-request.asciidoc @@ -64,9 +64,6 @@ query. |`analyzer` |The analyzer name to be used when analyzing the query string. -|`lowercase_expanded_terms` |Should terms be automatically lowercased or -not. Defaults to `true`. - |`analyze_wildcard` |Should wildcard and prefix queries be analyzed or not. Defaults to `false`. diff --git a/docs/reference/search/validate.asciidoc b/docs/reference/search/validate.asciidoc index 5fb4ad9b7ce15..5b015f4e5786b 100644 --- a/docs/reference/search/validate.asciidoc +++ b/docs/reference/search/validate.asciidoc @@ -52,9 +52,6 @@ query. |`lenient` |If set to true will cause format based failures (like providing text to a numeric field) to be ignored. Defaults to false. -|`lowercase_expanded_terms` |Should terms be automatically lowercased or -not. Defaults to `true`. - |`analyze_wildcard` |Should wildcard and prefix queries be analyzed or not. Defaults to `false`. |======================================================================= diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/count.json b/rest-api-spec/src/main/resources/rest-api-spec/api/count.json index 9048f982712b0..0e2697cd524d2 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/count.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/count.json @@ -67,10 +67,6 @@ "lenient": { "type" : "boolean", "description" : "Specify whether format-based query failures (such as providing text to a numeric field) should be ignored" - }, - "lowercase_expanded_terms": { - "type" : "boolean", - "description" : "Specify whether query terms should be lowercased" } } }, diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/delete_by_query.json b/rest-api-spec/src/main/resources/rest-api-spec/api/delete_by_query.json index f97492aa7ab2d..39998fb87fe27 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/delete_by_query.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/delete_by_query.json @@ -65,10 +65,6 @@ "type" : "boolean", "description" : "Specify whether format-based query failures (such as providing text to a numeric field) should be ignored" }, - "lowercase_expanded_terms": { - "type" : "boolean", - "description" : "Specify whether query terms should be lowercased" - }, "preference": { "type" : "string", "description" : "Specify the node or shard the operation should be performed on (default: random)" diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/explain.json b/rest-api-spec/src/main/resources/rest-api-spec/api/explain.json index 328794ffdd02d..0f0d8c132b395 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/explain.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/explain.json @@ -49,10 +49,6 @@ "type" : "boolean", "description" : "Specify whether format-based query failures (such as providing text to a numeric field) should be ignored" }, - "lowercase_expanded_terms": { - "type" : "boolean", - "description" : "Specify whether query terms should be lowercased" - }, "parent": { "type" : "string", "description" : "The ID of the parent document" diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.validate_query.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.validate_query.json index 98af689833a06..7a0977da194f7 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.validate_query.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.validate_query.json @@ -63,10 +63,6 @@ "type" : "boolean", "description" : "Specify whether format-based query failures (such as providing text to a numeric field) should be ignored" }, - "lowercase_expanded_terms": { - "type" : "boolean", - "description" : "Specify whether query terms should be lowercased" - }, "rewrite": { "type": "boolean", "description": "Provide a more detailed explanation showing the actual Lucene query that will be executed." diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/search.json b/rest-api-spec/src/main/resources/rest-api-spec/api/search.json index 5aa7a409a067a..2cf359ede16d2 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/search.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/search.json @@ -72,10 +72,6 @@ "type" : "boolean", "description" : "Specify whether format-based query failures (such as providing text to a numeric field) should be ignored" }, - "lowercase_expanded_terms": { - "type" : "boolean", - "description" : "Specify whether query terms should be lowercased" - }, "preference": { "type" : "string", "description" : "Specify the node or shard the operation should be performed on (default: random)" diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/update_by_query.json b/rest-api-spec/src/main/resources/rest-api-spec/api/update_by_query.json index 4b9e76ac59f16..afa2a79570c24 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/update_by_query.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/update_by_query.json @@ -65,10 +65,6 @@ "type" : "boolean", "description" : "Specify whether format-based query failures (such as providing text to a numeric field) should be ignored" }, - "lowercase_expanded_terms": { - "type" : "boolean", - "description" : "Specify whether query terms should be lowercased" - }, "pipeline": { "type" : "string", "description" : "Ingest pipeline to set on index requests made by this action. (default: none)" diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/count/20_query_string.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/count/20_query_string.yaml index 933033761e9c1..70f402691a379 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/count/20_query_string.yaml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/count/20_query_string.yaml @@ -58,15 +58,6 @@ count: index: test q: field:BA* - lowercase_expanded_terms: false - - - match: {count : 0} - - - do: - count: - index: test - q: field:BA* - analyze_wildcard: true - match: {count : 1} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/explain/30_query_string.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/explain/30_query_string.yaml index 30fe6cc55b621..78ef8c4bc8942 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/explain/30_query_string.yaml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/explain/30_query_string.yaml @@ -68,17 +68,6 @@ type: test id: 1 q: field:BA* - lowercase_expanded_terms: false - - - is_false: matched - - - do: - explain: - index: test - type: test - id: 1 - q: field:BA* - analyze_wildcard: true - is_true: matched diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.validate_query/20_query_string.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.validate_query/20_query_string.yaml index c6dd323aa6c98..3f96009c12a6c 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.validate_query/20_query_string.yaml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.validate_query/20_query_string.yaml @@ -49,15 +49,6 @@ indices.validate_query: index: test q: field:BA* - lowercase_expanded_terms: false - - - is_true: valid - - - do: - indices.validate_query: - index: test - q: field:BA* - analyze_wildcard: true - is_true: valid diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/60_query_string.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/60_query_string.yaml index 6fb93bb10449a..8533cfd26686d 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/60_query_string.yaml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/60_query_string.yaml @@ -58,15 +58,6 @@ search: index: test q: field:BA* - lowercase_expanded_terms: false - - - match: {hits.total: 0} - - - do: - search: - index: test - q: field:BA* - analyze_wildcard: true - match: {hits.total: 1}