Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Exclude unmapped fields during max clause limit checking for querying #49523

Merged
merged 12 commits into from
Jan 20, 2020
Merged

Exclude unmapped fields during max clause limit checking for querying #49523

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
81b1256
Exclude unmapped fileds during max clause limit checking for querying
zacharymorn Nov 21, 2019
79df51c
Address comments
zacharymorn Nov 26, 2019
4268745
Remove unneeded changes in elasticsearch.yml
zacharymorn Nov 28, 2019
b4279d1
Add integration test for igoring unmapping fields
zacharymorn Jan 10, 2020
60efd2c
Remove unit tests and unneeded changes to integration test
zacharymorn Jan 14, 2020
abbb18d
Merge branch 'master' into issue-49002
elasticmachine Jan 15, 2020
0c72d0b
Merge branch 'master' into issue-49002
zacharymorn Jan 16, 2020
fe53f2d
Update test to use CreateIndexRequestBuilder.setMapping after merge f…
zacharymorn Jan 16, 2020
6804d83
Merge branch 'master' into issue-49002
zacharymorn Jan 16, 2020
96436a3
Merge branch 'issue-49002' of https://github.com/zacharymorn/elastics…
zacharymorn Jan 16, 2020
a6ef57d
ix test failure as type has been removed in CreateIndexRequest
zacharymorn Jan 16, 2020
4ef62f5
Merge branch 'master' into issue-49002
elasticmachine Jan 17, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 3 additions & 4 deletions server/src/main/java/org/elasticsearch/index/search/QueryParserHelper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,8 +87,8 @@ public static Map<String, Float> resolveMappingFields(QueryShardContext context,
boolean allField = Regex.isMatchAllPattern(fieldEntry.getKey());
boolean multiField = Regex.isSimpleMatchPattern(fieldEntry.getKey());
float weight = fieldEntry.getValue() == null ? 1.0f : fieldEntry.getValue();
Map<String, Float> fieldMap = resolveMappingField(context, fieldEntry.getKey(), weight,
!multiField, !allField, fieldSuffix);
Map<String, Float> fieldMap = resolveMappingField(context, fieldEntry.getKey(), weight, !multiField, !allField, fieldSuffix);

for (Map.Entry<String, Float> field : fieldMap.entrySet()) {
float boost = field.getValue();
if (resolvedFields.containsKey(field.getKey())) {
Expand All @@ -97,6 +97,7 @@ public static Map<String, Float> resolveMappingFields(QueryShardContext context,
resolvedFields.put(field.getKey(), boost);
}
}

checkForTooManyFields(resolvedFields, context);
return resolvedFields;
}
Expand Down Expand Up @@ -141,8 +142,6 @@ public static Map<String, Float> resolveMappingField(QueryShardContext context,

MappedFieldType fieldType = context.getMapperService().fullName(fieldName);
if (fieldType == null) {
// Note that we don't ignore unmapped fields.
fields.put(fieldName, weight);
continue;
}

Expand Down
64 changes: 64 additions & 0 deletions server/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@
import org.apache.lucene.util.automaton.Automaton;
import org.apache.lucene.util.automaton.Operations;
import org.apache.lucene.util.automaton.TooComplexToDeterminizeException;
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.compress.CompressedXContent;
Expand Down Expand Up @@ -399,6 +400,14 @@ protected void doAssertLuceneQuery(QueryStringQueryBuilder queryBuilder,
// nothing yet, put additional assertions here.
}

@Override
protected Settings createTestIndexSettings() {
return Settings.builder()
.put(super.createTestIndexSettings())
.put("indices.query.bool.max_clause_count", "13")
.build();
}

// Tests fix for https://github.com/elastic/elasticsearch/issues/29403
public void testTimezoneEquals() {
QueryStringQueryBuilder builder1 = new QueryStringQueryBuilder("bar");
Expand Down Expand Up @@ -1290,6 +1299,61 @@ public void testQuoteAnalyzer() throws Exception {
assertEquals(expectedQuery, query);
}

// The only expectation for this test is to not throw exception
public void testToQueryExceedingMaxClauseCountWithUnmappedFields() throws Exception {
QueryShardContext shardContext = createShardContext();

new QueryStringQueryBuilder("\"bar\"")
.field("*")
.field("unmappedField1")
.field("unmappedField2")
.field("unmappedField3")
.field("unmappedField4")
.toQuery(shardContext);
}

public void testToQueryMaxClauseWithWeight() throws IOException {
QueryShardContext shardContext = createShardContext();
shardContext.getMapperService()
.merge("_doc", new CompressedXContent(Strings.toString(PutMappingRequest.buildFromSimplifiedDef("_doc",
"additionalMappedField1", "type=text",
"additionalMappedField2", "type=text",
"additionalMappedField3", "type=text",
"additionalMappedField4", "type=text"
))), MapperService.MergeReason.MAPPING_UPDATE);

try {
new QueryStringQueryBuilder("\"bar\"")
.field("*")
.toQuery(shardContext);
fail();
}
catch (IllegalArgumentException e) {
assertEquals("field expansion matches too many fields, limit: 13, got: 17", e.getMessage());
}

}

public void testToQueryMaxClauseNoWeight() throws IOException {
QueryShardContext shardContext = createShardContext();
shardContext.getMapperService()
.merge("_doc", new CompressedXContent(Strings.toString(PutMappingRequest.buildFromSimplifiedDef("_doc",
"additionalMappedField1", "type=text",
"additionalMappedField2", "type=text",
"additionalMappedField3", "type=text",
"additionalMappedField4", "type=text"
))), MapperService.MergeReason.MAPPING_UPDATE);

try {
new QueryStringQueryBuilder("\"bar\"")
.toQuery(shardContext);
fail();
}
catch (IllegalArgumentException e) {
assertEquals("field expansion matches too many fields, limit: 13, got: 17", e.getMessage());
}
}

public void testQuoteFieldSuffix() throws IOException {
QueryShardContext context = createShardContext();
assertEquals(new TermQuery(new Term(STRING_FIELD_NAME, "bar")),
Expand Down
1 change: 0 additions & 1 deletion server/src/test/resources/config/elasticsearch.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,2 @@

yaml.config.exists: true

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

can you revert this change

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah missed this, removed.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It seems that it's still there ? ;)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hmm that's strange, might be my IDE setup problem. Anyway I checked out and committed the previous version of this file before my changes, and I don't see this change in Files Changed tab anymore. so it should be good now.

30 changes: 13 additions & 17 deletions test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -212,21 +212,17 @@ public static void afterClass() throws Exception {

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Changes on this file are not needed. Can you revert ?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sorry I forgot to add a comment for this. The changes here are because in two of the new test cases I added, the state of shardContext.mapperService was changed to force the condition, which affected other test cases as well. By re-creating serviceHolder for every test case, side effects from test case does not affect another.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why is it not possible to add your extra fields in QueryStringQueryBuilderTests#initializeAdditionalMappings ? That's where we expect tests to modify the shared mapping. Can you show the test failures if that doesn't work ?

Copy link
Contributor Author

@zacharymorn zacharymorn Nov 28, 2019 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I did look at that method before. I think the problem of adding additional fields in QueryStringQueryBuilderTests#initializeAdditionalMappings was that it is adding additional mappings for ALL test cases in that class (the additional xContent mapping is hard coded in the method, which is an override one as well so I can't change its interface to take in additional parameter easily), so that may cause other test cases in that class to also fail.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sorry for the delay. I tried to modify QueryStringQueryBuilderTests#initializeAdditionalMappings like the following:

@Override
    protected void initializeAdditionalMappings(MapperService mapperService) throws IOException {
        XContentBuilder mapping = jsonBuilder().startObject().startObject("_doc").startObject("properties")
            .startObject("prefix_field")
                .field("type", "text")
                    .startObject("index_prefixes").endObject()
                .endObject()
            .startObject("additionalMappedField1")
                .field("type", "text")
            .endObject()
            .startObject("additionalMappedField2")
                .field("type", "text")
            .endObject()
            .startObject("additionalMappedField3")
                .field("type", "text")
            .endObject()
            .startObject("additionalMappedField4")
                .field("type", "text")
            .endObject()
            .endObject().endObject().endObject();

        mapperService.merge("_doc",
            new CompressedXContent(Strings.toString(mapping)), MapperService.MergeReason.MAPPING_UPDATE);
    }

and all tests worked without modification. Could you try so that we can remove this unneeded change ?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes sorry I checked without the modified indices.query.bool.max_clause_count. It seems tricky to test this behavior here so maybe you can modify or add a new test similar to

elasticsearch/server/src/test/java/org/elasticsearch/search/query/QueryStringIT.java

Line 266 in a59b065

public void testLimitOnExpandedFields() throws Exception {
?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Didn’t realize there’s an integration test for this as well…I just added a new test there.

I also removed this line of Index setting

elasticsearch/server/src/test/java/org/elasticsearch/search/query/QueryStringIT.java

Lines 279 to 280 in a59b065

.setSettings(Settings.builder().put(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(),
CLUSTER_MAX_CLAUSE_COUNT + 100))
in test QueryStringIT.testLimitOnExpandedFields as it doesn’t seems to make sense in the context (why is it setting INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING to CLUSTER_MAX_CLAUSE_COUNT + 100?), and have no impact to test result. But please let me know if that’s indeed needed and I can put it back.

Having added the additional test though, I feel this test and the ones I’ve already put in and passing are different levels of tests (integration & unit tests) and thus it doesn’t hurt for them to coexist. Are you suggesting to have integration test only for these changes?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Are you suggesting to have integration test only for these changes?

Yes, mainly because it seems challenging to have a simple unit test for this and we already have integration tests.

In test QueryStringIT.testLimitOnExpandedFields as it doesn’t seems to make sense in the context (why is it setting INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING to CLUSTER_MAX_CLAUSE_COUNT + 100?), and have no impact to test result. But please let me know if that’s indeed needed and I can put it back.

I don't think it's needed either but that's unrelated to this pr so not sure if we should remove it now ?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I just pushed another commit that reverted the unneeded change to the integration test, and the unit tests and their related changes.

However, I guess I’m still a bit confused for why it’s tricky to keep those unit tests there, when they are already passing and verifying the right logic. As can be seen in the new (reverting) commit, the changes I made to allow unit tests there are the following:

  1. The new unit tests themselves
  2. Local, test class level change to force the condition for the added unit tests (setting indices.query.bool.max_clause_count)
  3. Super test class level change to make sure that each tests are independent to each other so that Discovery: Support local (JVM level) discovery #2 doesn’t break other test cases (the changes in AbstractBuilderTestCase.beforeTest)

These changes combined would allow the newly added test cases to pass, and not break any existing test cases.

I feel I’m still having some gap in understanding the reasoning here, but I’ll leave it to you. Thanks for your patience and time for reviewing my changes!

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I feel I’m still having some gap in understanding the reasoning here, but I’ll leave it to you

My reasoning here is that the abstract class for these tests was designed to share the mapping within the tests so it would require a bigger refactoring to change this behavior. We also have a lot of unit tests here so even if they pass I am cautious of the additional work that would be needed if we switch to one configuration per test.

@Before
public void beforeTest() throws Exception {
if (serviceHolder == null) {
assert serviceHolderWithNoType == null;
// we initialize the serviceHolder and serviceHolderWithNoType just once, but need some
// calls to the randomness source during its setup. In order to not mix these calls with
// the randomness source that is later used in the test method, we use the master seed during
// this setup
long masterSeed = SeedUtils.parseSeed(RandomizedTest.getContext().getRunnerSeedAsString());
RandomizedTest.getContext().runWithPrivateRandomness(masterSeed, (Callable<Void>) () -> {
serviceHolder = new ServiceHolder(nodeSettings, createTestIndexSettings(), getPlugins(), nowInMillis,
AbstractBuilderTestCase.this, true);
serviceHolderWithNoType = new ServiceHolder(nodeSettings, createTestIndexSettings(), getPlugins(), nowInMillis,
AbstractBuilderTestCase.this, false);
return null;
});
}
// we re-initiate serviceHolder for every test as some test cases may alter the state of serviceHolder
// and its inner objects, and reusing the same one may cause failure of other test cases.
// E.g. some test cases may alter field mapping information in serviceHolder.mapperService
long masterSeed = SeedUtils.parseSeed(RandomizedTest.getContext().getRunnerSeedAsString());
RandomizedTest.getContext().runWithPrivateRandomness(masterSeed, (Callable<Void>) () -> {
serviceHolder = new ServiceHolder(nodeSettings, createTestIndexSettings(), getPlugins(), nowInMillis,
AbstractBuilderTestCase.this, true);
serviceHolderWithNoType = new ServiceHolder(nodeSettings, createTestIndexSettings(), getPlugins(), nowInMillis,
AbstractBuilderTestCase.this, false);
return null;
});

serviceHolder.clientInvocationHandler.delegate = this;
serviceHolderWithNoType.clientInvocationHandler.delegate = this;
Expand Down Expand Up @@ -400,11 +396,11 @@ public void onRemoval(ShardId shardId, Accountable accountable) {
testCase.initializeAdditionalMappings(mapperService);
}
}

public static Predicate<String> indexNameMatcher() {
// Simplistic index name matcher used for testing
return pattern -> Regex.simpleMatch(pattern, index.getName());
}
}

@Override
public void close() throws IOException {
Expand Down