Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Remove the lowercase_expanded_terms and locale options from (simple_)query_string. #19057

Closed
wants to merge 1 commit into from
Closed

Remove the lowercase_expanded_terms and locale options from (simple_)query_string. #19057

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
189 changes: 102 additions & 87 deletions core/src/main/java/org/apache/lucene/queryparser/classic/MapperQueryParser.java
View file
Open in desktop
Expand Up @@ -93,18 +93,41 @@ public void reset(QueryParserSettings settings) {
} else {
this.field = null;
}
setAnalyzer(settings.analyzer());
setMultiTermRewriteMethod(settings.rewriteMethod());
if (settings.analyzer() != null) {
setAnalyzer(settings.analyzer());
} else {
setAnalyzer(context.getMapperService().searchAnalyzer());
}
if (settings.rewriteMethod() != null) {
setMultiTermRewriteMethod(settings.rewriteMethod());
}
setEnablePositionIncrements(settings.enablePositionIncrements());
setAutoGeneratePhraseQueries(settings.autoGeneratePhraseQueries());
setMaxDeterminizedStates(settings.maxDeterminizedStates());
setAllowLeadingWildcard(settings.allowLeadingWildcard());
setLowercaseExpandedTerms(settings.lowercaseExpandedTerms());
setLowercaseExpandedTerms(false); // no need for it, we use the mappings to figure it out
setPhraseSlop(settings.phraseSlop());
setDefaultOperator(settings.defaultOperator());
setFuzzyMinSim(settings.fuzziness().asFloat());
setFuzzyPrefixLength(settings.fuzzyPrefixLength());
setLocale(settings.locale());
}

private void setMultiTermAnalyzer() {
if (settings.multiTermAnalyzer() != null) {
setAnalyzer(settings.multiTermAnalyzer());
} else {
setAnalyzer(context.getMapperService().searchMultiTermAnalyzer());
}
}

private void setQuoteAnalyzer() {
if (settings.quoteAnalyzer() != null) {
setAnalyzer(settings.quoteAnalyzer());
} else if (settings.analyzer() != null) {
setAnalyzer(settings.analyzer());
} else {
setAnalyzer(context.getMapperService().searchQuoteAnalyzer());
}
}

/**
Expand Down Expand Up @@ -197,31 +220,21 @@ private Query getFieldQuerySingle(String field, String queryText, boolean quoted
currentFieldType = null;
Analyzer oldAnalyzer = getAnalyzer();
try {
if (quoted) {
setAnalyzer(settings.quoteAnalyzer());
if (settings.quoteFieldSuffix() != null) {
currentFieldType = context.fieldMapper(field + settings.quoteFieldSuffix());
}
if (quoted && settings.quoteFieldSuffix() != null) {
currentFieldType = context.fieldMapper(field + settings.quoteFieldSuffix());
}
if (currentFieldType == null) {
currentFieldType = context.fieldMapper(field);
}
if (currentFieldType != null) {
if (quoted) {
if (!settings.forceQuoteAnalyzer()) {
setAnalyzer(context.getSearchQuoteAnalyzer(currentFieldType));
}
} else {
if (!settings.forceAnalyzer()) {
setAnalyzer(context.getSearchAnalyzer(currentFieldType));
}
setQuoteAnalyzer();
}
if (currentFieldType != null) {
Query query = null;
if (currentFieldType.tokenized() == false) {
// this might be a structured field like a numeric
try {
query = currentFieldType.termQuery(queryText, context);
return currentFieldType.termQuery(queryText, context);
} catch (RuntimeException e) {
if (settings.lenient()) {
return null;
Expand All @@ -230,10 +243,6 @@ private Query getFieldQuerySingle(String field, String queryText, boolean quoted
}
}
}
if (query == null) {
query = super.getFieldQuery(currentFieldType.name(), queryText, quoted);
}
return query;
}
}
return super.getFieldQuery(field, queryText, quoted);
Expand Down Expand Up @@ -328,39 +337,61 @@ protected Query getRangeQuery(String field, String part1, String part2,

private Query getRangeQuerySingle(String field, String part1, String part2,
boolean startInclusive, boolean endInclusive) {
currentFieldType = context.fieldMapper(field);
if (currentFieldType != null) {
if (lowercaseExpandedTerms && currentFieldType.tokenized()) {
part1 = part1 == null ? null : part1.toLowerCase(locale);
part2 = part2 == null ? null : part2.toLowerCase(locale);
}
Analyzer oldAnalyzer = getAnalyzer();
try {
setMultiTermAnalyzer();
currentFieldType = context.fieldMapper(field);
if (currentFieldType != null) {
try {
if (currentFieldType.tokenized()) {
part1 = part1 == null ? null : analyzeSingleToken(field, part1);
part2 = part2 == null ? null : analyzeSingleToken(field, part2);
}

try {
Query rangeQuery;
if (currentFieldType instanceof LegacyDateFieldMapper.DateFieldType && settings.timeZone() != null) {
LegacyDateFieldMapper.DateFieldType dateFieldType = (LegacyDateFieldMapper.DateFieldType) this.currentFieldType;
rangeQuery = dateFieldType.rangeQuery(part1, part2, startInclusive, endInclusive, settings.timeZone(), null);
} else if (currentFieldType instanceof DateFieldMapper.DateFieldType && settings.timeZone() != null) {
DateFieldMapper.DateFieldType dateFieldType = (DateFieldMapper.DateFieldType) this.currentFieldType;
rangeQuery = dateFieldType.rangeQuery(part1, part2, startInclusive, endInclusive, settings.timeZone(), null);
} else {
rangeQuery = currentFieldType.rangeQuery(part1, part2, startInclusive, endInclusive);
}
return rangeQuery;
} catch (RuntimeException e) {
if (settings.lenient()) {
return null;
Query rangeQuery;
if (currentFieldType instanceof LegacyDateFieldMapper.DateFieldType && settings.timeZone() != null) {
LegacyDateFieldMapper.DateFieldType dateFieldType = (LegacyDateFieldMapper.DateFieldType) this.currentFieldType;
rangeQuery = dateFieldType.rangeQuery(part1, part2, startInclusive, endInclusive, settings.timeZone(), null);
} else if (currentFieldType instanceof DateFieldMapper.DateFieldType && settings.timeZone() != null) {
DateFieldMapper.DateFieldType dateFieldType = (DateFieldMapper.DateFieldType) this.currentFieldType;
rangeQuery = dateFieldType.rangeQuery(part1, part2, startInclusive, endInclusive, settings.timeZone(), null);
} else {
rangeQuery = currentFieldType.rangeQuery(part1, part2, startInclusive, endInclusive);
}
return rangeQuery;
} catch (RuntimeException e) {
if (settings.lenient()) {
return null;
}
throw e;
}
throw e;
}
return newRangeQuery(field, part1, part2, startInclusive, endInclusive);
} finally {
setAnalyzer(oldAnalyzer);
}
return newRangeQuery(field, part1, part2, startInclusive, endInclusive);
}

protected Query getFuzzyQuery(String field, String termStr, String minSimilarity) throws ParseException {
if (lowercaseExpandedTerms) {
termStr = termStr.toLowerCase(locale);
private String analyzeSingleToken(String field, String value) {
try (TokenStream tk = getAnalyzer().tokenStream(field, value)) {
CharTermAttribute termAtt = tk.addAttribute(CharTermAttribute.class);
tk.reset();
final String term;
if (tk.incrementToken()) {
term = termAtt.toString();
} else {
throw new IllegalStateException("Expected 1 token but got 0");
}
if (tk.incrementToken()) {
throw new IllegalStateException("Expected 1 token but got 2 or more");
}
return term;
} catch (IOException e) {
throw new IllegalStateException("Cannot happen", e);
}
}

protected Query getFuzzyQuery(String field, String termStr, String minSimilarity) throws ParseException {
Collection<String> fields = extractMultiFields(field);
if (fields != null) {
if (fields.size() == 1) {
Expand Down Expand Up @@ -396,19 +427,26 @@ protected Query getFuzzyQuery(String field, String termStr, String minSimilarity
}

private Query getFuzzyQuerySingle(String field, String termStr, String minSimilarity) throws ParseException {
currentFieldType = context.fieldMapper(field);
if (currentFieldType != null) {
try {
return currentFieldType.fuzzyQuery(termStr, Fuzziness.build(minSimilarity),
fuzzyPrefixLength, settings.fuzzyMaxExpansions(), FuzzyQuery.defaultTranspositions);
} catch (RuntimeException e) {
if (settings.lenient()) {
return null;
Analyzer oldAnalyzer = getAnalyzer();
try {
setMultiTermAnalyzer();
currentFieldType = context.fieldMapper(field);
if (currentFieldType != null) {
try {
termStr = termStr == null ? null : analyzeSingleToken(field, termStr);
return currentFieldType.fuzzyQuery(termStr, Fuzziness.build(minSimilarity),
fuzzyPrefixLength, settings.fuzzyMaxExpansions(), FuzzyQuery.defaultTranspositions);
} catch (RuntimeException e) {
if (settings.lenient()) {
return null;
}
throw e;
}
throw e;
}
return super.getFuzzyQuery(field, termStr, Float.parseFloat(minSimilarity));
} finally {
setAnalyzer(oldAnalyzer);
}
return super.getFuzzyQuery(field, termStr, Float.parseFloat(minSimilarity));
}

@Override
Expand All @@ -423,9 +461,6 @@ protected Query newFuzzyQuery(Term term, float minimumSimilarity, int prefixLeng

@Override
protected Query getPrefixQuery(String field, String termStr) throws ParseException {
if (lowercaseExpandedTerms) {
termStr = termStr.toLowerCase(locale);
}
Collection<String> fields = extractMultiFields(field);
if (fields != null) {
if (fields.size() == 1) {
Expand Down Expand Up @@ -467,9 +502,6 @@ private Query getPrefixQuerySingle(String field, String termStr) throws ParseExc
try {
currentFieldType = context.fieldMapper(field);
if (currentFieldType != null) {
if (!settings.forceAnalyzer()) {
setAnalyzer(context.getSearchAnalyzer(currentFieldType));
}
Query query = null;
if (currentFieldType.tokenized() == false) {
query = currentFieldType.prefixQuery(termStr, multiTermRewriteMethod, context);
Expand All @@ -491,8 +523,8 @@ private Query getPrefixQuerySingle(String field, String termStr) throws ParseExc
}

private Query getPossiblyAnalyzedPrefixQuery(String field, String termStr) throws ParseException {
if (!settings.analyzeWildcard()) {
return super.getPrefixQuery(field, termStr);
if (settings.analyzeWildcard() == false) {
setMultiTermAnalyzer();
}
List<List<String> > tlist;
// get Analyzer from superclass and tokenize the term
Expand Down Expand Up @@ -590,9 +622,6 @@ protected Query getWildcardQuery(String field, String termStr) throws ParseExcep
return FIELD_QUERY_EXTENSIONS.get(ExistsFieldQueryExtension.NAME).query(context, actualField);
}
}
if (lowercaseExpandedTerms) {
termStr = termStr.toLowerCase(locale);
}
Collection<String> fields = extractMultiFields(field);
if (fields != null) {
if (fields.size() == 1) {
Expand Down Expand Up @@ -629,19 +658,10 @@ protected Query getWildcardQuery(String field, String termStr) throws ParseExcep
}

private Query getWildcardQuerySingle(String field, String termStr) throws ParseException {
String indexedNameField = field;
currentFieldType = null;
Analyzer oldAnalyzer = getAnalyzer();
try {
currentFieldType = context.fieldMapper(field);
if (currentFieldType != null) {
if (!settings.forceAnalyzer()) {
setAnalyzer(context.getSearchAnalyzer(currentFieldType));
}
indexedNameField = currentFieldType.name();
return getPossiblyAnalyzedWildcardQuery(indexedNameField, termStr);
}
return getPossiblyAnalyzedWildcardQuery(indexedNameField, termStr);
return getPossiblyAnalyzedWildcardQuery(field, termStr);
} catch (RuntimeException e) {
if (settings.lenient()) {
return null;
Expand All @@ -653,8 +673,8 @@ private Query getWildcardQuerySingle(String field, String termStr) throws ParseE
}

private Query getPossiblyAnalyzedWildcardQuery(String field, String termStr) throws ParseException {
if (!settings.analyzeWildcard()) {
return super.getWildcardQuery(field, termStr);
if (settings.analyzeWildcard() == false) {
setMultiTermAnalyzer();
}
boolean isWithinToken = (!termStr.startsWith("?") && !termStr.startsWith("*"));
StringBuilder aggStr = new StringBuilder();
Expand Down Expand Up @@ -718,9 +738,6 @@ private Query getPossiblyAnalyzedWildcardQuery(String field, String termStr) thr

@Override
protected Query getRegexpQuery(String field, String termStr) throws ParseException {
if (lowercaseExpandedTerms) {
termStr = termStr.toLowerCase(locale);
}
Collection<String> fields = extractMultiFields(field);
if (fields != null) {
if (fields.size() == 1) {
Expand Down Expand Up @@ -760,11 +777,9 @@ private Query getRegexpQuerySingle(String field, String termStr) throws ParseExc
currentFieldType = null;
Analyzer oldAnalyzer = getAnalyzer();
try {
setMultiTermAnalyzer();
currentFieldType = context.fieldMapper(field);
if (currentFieldType != null) {
if (!settings.forceAnalyzer()) {
setAnalyzer(context.getSearchAnalyzer(currentFieldType));
}
Query query = null;
if (currentFieldType.tokenized() == false) {
query = currentFieldType.regexpQuery(termStr, RegExp.ALL,
Expand Down