Skip to content

Commit

Permalink
Removed redundant "bool" wrapper clauses.
Browse files Browse the repository at this point in the history
  • Loading branch information
@davemoore-
davemoore- committed Jul 18, 2019
1 parent 117f38f commit a1332ce
Show file tree
Hide file tree
Showing 4 changed files with 66 additions and 33 deletions.
2 changes: 1 addition & 1 deletion README.md
View file
Expand Up @@ -31,7 +31,7 @@ Once you have installed Elasticsearch, you can install zentity from a remote URL

Example:

`elasticsearch-plugin install https://zentity.io/releases/zentity-1.3.0-elasticsearch-7.2.0.zip`
`elasticsearch-plugin install https://zentity.io/releases/zentity-1.3.1-beta1-elasticsearch-7.2.0.zip`

Read the [installation](https://zentity.io/docs/installation) docs for more details.

Expand Down
2 changes: 1 addition & 1 deletion pom.xml
View file
Expand Up @@ -14,7 +14,7 @@
<zentity.author>Dave Moore</zentity.author>
<zentity.classname>org.elasticsearch.plugin.zentity.ZentityPlugin</zentity.classname>
<zentity.website>https://zentity.io</zentity.website>
<zentity.version>1.3.0</zentity.version>
<zentity.version>1.3.1-beta1</zentity.version>
<!-- dependency versions -->
<elasticsearch.version>7.2.0</elasticsearch.version>
<jackson.version>2.9.9</jackson.version>
Expand Down
93 changes: 63 additions & 30 deletions src/main/java/io/zentity/resolution/Job.java
View file
Expand Up @@ -288,9 +288,11 @@ public static List<String> makeIndexFieldClauses(Model model, String indexName,
continue;

// Combine each value clause into a single "should" or "filter" clause.
String valuesClause = String.join(",", valueClauses);
String valuesClause;
if (valueClauses.size() > 1)
valuesClause = "{\"bool\":{\"" + combiner + "\":[" + valuesClause + "]}}";
valuesClause = "{\"bool\":{\"" + combiner + "\":[" + String.join(",", valueClauses) + "]}}";
else
valuesClause = valueClauses.get(0);
indexFieldClauses.add(valuesClause);
}
return indexFieldClauses;
Expand Down Expand Up @@ -319,9 +321,11 @@ public static List<String> makeAttributeClauses(Model model, String indexName, M
continue;

// Combine each matcher clause into a single "should" or "filter" clause.
String indexFieldsClause = String.join(",", indexFieldClauses);
String indexFieldsClause;
if (indexFieldClauses.size() > 1)
indexFieldsClause = "{\"bool\":{\"" + combiner + "\":[" + indexFieldsClause + "]}}";
indexFieldsClause = "{\"bool\":{\"" + combiner + "\":[" + String.join(",", indexFieldClauses) + "]}}";
else
indexFieldsClause = indexFieldClauses.get(0);
attributeClauses.add(indexFieldsClause);
}
return attributeClauses;
Expand All @@ -347,14 +351,16 @@ public static String populateResolversFilterTree(Model model, String indexName,
if (indexFieldClauses.size() == 0)
continue;

// Combine each matcher clause into a single "should" clause.
String indexFieldsClause = String.join(",", indexFieldClauses);
// Combine multiple matcher clauses into a single "should" clause.
String indexFieldsClause;
if (indexFieldClauses.size() > 1)
indexFieldsClause = "{\"bool\":{\"should\":[" + indexFieldsClause + "]}}";
indexFieldsClause = "{\"bool\":{\"should\":[" + String.join(",", indexFieldClauses) + "]}}";
else
indexFieldsClause = indexFieldClauses.get(0);

// Populate any child filters.
String filter = populateResolversFilterTree(model, indexName, resolversFilterTree.get(attributeName), attributes, includeExplanation, _nameIdCounter);
if (!filter.equals("{}"))
if (!filter.isEmpty())
attributeClauses.add("{\"bool\":{\"filter\":[" + indexFieldsClause + "," + filter + "]}}");
else
attributeClauses.add(indexFieldsClause);
Expand All @@ -366,9 +372,9 @@ public static String populateResolversFilterTree(Model model, String indexName,
if (size > 1)
return "{\"bool\":{\"should\":[" + String.join(",", attributeClauses) + "]}}";
else if (size == 1)
return "{\"bool\":{\"filter\":" + attributeClauses.get(0) + "}}";
return attributeClauses.get(0);
else
return "{}";
return "";
}

/**
Expand Down Expand Up @@ -601,10 +607,11 @@ private void traverse() throws IOException, ValidationException {

// Construct query for this index.
String query;
String queryClause = "{}";
List<String> queryClauses = new ArrayList<>();
String queryClause;
List<String> queryMustNotClauses = new ArrayList<>();
String queryMustNotClause = "";
List<String> queryFilterClauses = new ArrayList<>();
String queryFilterClause = "";
List<String> topLevelClauses = new ArrayList<>();
topLevelClauses.add("\"_source\":true");

Expand All @@ -624,8 +631,10 @@ else if (size == 1)
}

// Construct the top-level "must_not" clause.
if (!queryMustNotClauses.isEmpty())
queryClauses.add("\"must_not\":[" + String.join(",", queryMustNotClauses) + "]");
if (queryMustNotClauses.size() > 1)
queryMustNotClause = "\"must_not\":[" + String.join(",", queryMustNotClauses) + "]";
else if (queryMustNotClauses.size() == 1)
queryMustNotClause = "\"must_not\":" + queryMustNotClauses.get(0);

// Construct "scope.include.attributes" clauses. Combine them into a single "filter" clause.
if (!this.input.scope().include().attributes().isEmpty()) {
Expand All @@ -638,14 +647,14 @@ else if (size == 1)
}

// Construct the "ids" clause if this is the first hop and if any ids are specified for this index.
String idsClause = "{}";
String idsClause = "";
if (filterIds) {
Set<String> ids = this.input().ids().get(indexName);
idsClause = "{\"bool\":{\"filter\":[{\"ids\":{\"values\":[" + String.join(",", ids) + "]}}]}}";
}

// Construct the resolvers clause.
String resolversClause = "{}";
String resolversClause = "";
TreeMap<String, TreeMap> resolversFilterTree;
TreeMap<Integer, TreeMap<String, TreeMap>> resolversFilterTreeGrouped= new TreeMap<>(Collections.reverseOrder());
if (!this.attributes.isEmpty()) {
Expand Down Expand Up @@ -695,35 +704,59 @@ else if (size == 1)
// Construct a "should" clause for the above two clauses.
parentResolverClauses.add("{\"bool\":{\"should\":[" + attributesExistsClause + "," + parentResolverClause + "]}}");
}

// Construct a "filter" clause for every higher weight resolver clause.
parentResolversClauses.add("{\"bool\":{\"filter\":[" + String.join(",", parentResolverClauses) + "]}}");
if (parentResolverClauses.size() > 1)
parentResolversClauses.add("{\"bool\":{\"filter\":[" + String.join(",", parentResolverClauses) + "]}}");
else if (parentResolverClauses.size() == 1)
parentResolversClauses.add(parentResolverClauses.get(0));
}
}

// Combine the resolvers clause and parent resolvers clause in a "filter" query if necessary.
if (parentResolversClauses.size() > 0)
resolversClause = "{\"bool\":{\"filter\":[" + resolversClause + "," + String.join(",", parentResolversClauses) + "]}}";
}
}

// Combine the ids clause and resolvers clause in a "should" clause if necessary.
if (!idsClause.equals("{}") && !resolversClause.equals("{}"))
if (!idsClause.isEmpty() && !resolversClause.isEmpty())
queryFilterClauses.add("{\"bool\":{\"should\":[" + idsClause + "," + resolversClause + "]}}");
else if (!idsClause.equals("{}"))
else if (!idsClause.isEmpty())
queryFilterClauses.add(idsClause);
else
else if (!resolversClause.isEmpty())
queryFilterClauses.add(resolversClause);

// Construct the top-level "filter" clause.
if (!queryFilterClauses.isEmpty()) {
// Construct the "query" clause.
if (!queryMustNotClause.isEmpty() && queryFilterClauses.size() > 0) {

// Construct the top-level "filter" clause. Combine this clause and the top-level "must_not" clause
// in a "bool" clause and add it to the "query" field.
if (queryFilterClauses.size() > 1)
queryClauses.add("\"filter\":[" + String.join(",", queryFilterClauses) + "]");
queryFilterClause = "\"filter\":[" + String.join(",", queryFilterClauses) + "]";
else
queryClauses.add("\"filter\":" + queryFilterClauses.get(0));
}
queryFilterClause = "\"filter\":" + queryFilterClauses.get(0);
queryClause = "\"query\":{\"bool\":{" + queryMustNotClause + "," + queryFilterClause + "}}";

// Construct the "query" clause.
if (!queryClauses.isEmpty())
queryClause = "\"query\":{\"bool\":{" + String.join(",", queryClauses) + "}}";
} else if (!queryMustNotClause.isEmpty()) {

// Wrap only the top-level "must_not" clause in a "bool" clause and add it to the "query" field.
queryClause = "\"query\":{\"bool\":{" + queryMustNotClause + "}}";

} else if (queryFilterClauses.size() > 0) {

// Construct the top-level "filter" clause and add only this clause to the "query" field.
// This prevents a redundant "bool"."filter" wrapper clause when the top-level "must_not" clause
// does not exist.
if (queryFilterClauses.size() > 1)
queryFilterClause = "{\"bool\":{\"filter\":[" + String.join(",", queryFilterClauses) + "]}}";
else
queryFilterClause = queryFilterClauses.get(0);
queryClause = "\"query\":" + queryFilterClause;

} else {

// This should never be reached, and if somehow it did, Elasticsearch would return an error.
queryClause = "\"query\":{}";
}
topLevelClauses.add(queryClause);

// Construct the "script_fields" clause.
Expand Down
2 changes: 1 addition & 1 deletion src/test/java/io/zentity/resolution/JobTest.java
View file
Expand Up @@ -43,7 +43,7 @@ public void testMakeResolversClause() throws Exception {
List<List<String>> resolversSorted = Job.sortResolverAttributes(model, resolversList, counts);
TreeMap<String, TreeMap> resolversFilterTree = Job.makeResolversFilterTree(resolversSorted);
String resolversClause = Job.populateResolversFilterTree(model, "index", resolversFilterTree, input.attributes(), false, new AtomicInteger());
String expected = "{\"bool\":{\"should\":[{\"match\":{\"id\":\"1234567890\",\"fuzziness\":\"auto\"}},{\"bool\":{\"filter\":[{\"bool\":{\"should\":[{\"term\":{\"name\":\"Alice Jones\"}},{\"term\":{\"name\":\"Alice Jones-Smith\"}}]}},{\"bool\":{\"should\":[{\"match\":{\"phone\":\"555-123-4567\",\"fuzziness\":\"2\"}},{\"bool\":{\"filter\":[{\"term\":{\"street\":\"123 Main St\"}},{\"bool\":{\"should\":[{\"bool\":{\"filter\":[{\"term\":{\"city\":\"Beverly Hills\"}},{\"bool\":{\"filter\":{\"term\":{\"state\":\"CA\"}}}}]}},{\"term\":{\"zip\":\"90210\"}}]}}]}}]}}]}}]}}";
String expected = "{\"bool\":{\"should\":[{\"match\":{\"id\":\"1234567890\",\"fuzziness\":\"auto\"}},{\"bool\":{\"filter\":[{\"bool\":{\"should\":[{\"term\":{\"name\":\"Alice Jones\"}},{\"term\":{\"name\":\"Alice Jones-Smith\"}}]}},{\"bool\":{\"should\":[{\"match\":{\"phone\":\"555-123-4567\",\"fuzziness\":\"2\"}},{\"bool\":{\"filter\":[{\"term\":{\"street\":\"123 Main St\"}},{\"bool\":{\"should\":[{\"bool\":{\"filter\":[{\"term\":{\"city\":\"Beverly Hills\"}},{\"term\":{\"state\":\"CA\"}}]}},{\"term\":{\"zip\":\"90210\"}}]}}]}}]}}]}}]}}";
Assert.assertEquals(resolversClause, expected);
}

Expand Down

0 comments on commit a1332ce

Please sign in to comment.