Skip to content

Commit

Permalink
Introduced the concept of resolver priority. When a resolution job us…
Browse files Browse the repository at this point in the history
…es resolvers with different priority levels, then the higher priority resolvers either must match the document or must not exist in the document.
  • Loading branch information
davemoore- committed Jul 4, 2019
1 parent 6ad47c4 commit a57958d
Show file tree
Hide file tree
Showing 4 changed files with 94 additions and 8 deletions.
2 changes: 1 addition & 1 deletion README.md
Expand Up @@ -31,7 +31,7 @@ Once you have installed Elasticsearch, you can install zentity from a remote URL

Example:

`elasticsearch-plugin install https://zentity.io/releases/zentity-1.0.3-elasticsearch-7.2.0.zip`
`elasticsearch-plugin install https://zentity.io/releases/zentity-1.1.0-elasticsearch-7.2.0.zip`

Read the [installation](https://zentity.io/docs/installation) docs for more details.

Expand Down
2 changes: 1 addition & 1 deletion pom.xml
Expand Up @@ -14,7 +14,7 @@
<zentity.author>Dave Moore</zentity.author>
<zentity.classname>org.elasticsearch.plugin.zentity.ZentityPlugin</zentity.classname>
<zentity.website>https://zentity.io</zentity.website>
<zentity.version>1.0.3</zentity.version>
<zentity.version>1.1.0</zentity.version>
<!-- dependency versions -->
<elasticsearch.version>7.2.0</elasticsearch.version>
<jackson.version>2.9.9</jackson.version>
Expand Down
19 changes: 18 additions & 1 deletion src/main/java/io/zentity/model/Resolver.java
Expand Up @@ -19,6 +19,7 @@ public class Resolver {

private final String name;
private Set<String> attributes = new TreeSet<>();
private int priority = 0;

public Resolver(String name, JsonNode json) throws ValidationException {
validateName(name);
Expand All @@ -40,6 +41,8 @@ public Set<String> attributes() {
return this.attributes;
}

public int priority () { return this.priority; }

public void attributes(JsonNode value) throws ValidationException {
validateAttributes(value);
Set<String> attributes = new TreeSet<>();
Expand All @@ -48,6 +51,11 @@ public void attributes(JsonNode value) throws ValidationException {
this.attributes = attributes;
}

public void priority(JsonNode value) throws ValidationException {
validatePriority(value);
this.priority = value.asInt();
}

private void validateName(String value) throws ValidationException {
if (Patterns.EMPTY_STRING.matcher(value).matches())
throw new ValidationException("'resolvers' has a resolver with an empty name.");
Expand All @@ -67,6 +75,11 @@ private void validateAttributes(JsonNode value) throws ValidationException {
}
}

private void validatePriority(JsonNode value) throws ValidationException {
if (!value.isInt())
throw new ValidationException("'resolvers." + this.name + ".priority' must be an integer.");
}

private void validateObject(JsonNode object) throws ValidationException {
if (!object.isObject())
throw new ValidationException("'resolvers." + this.name + "' must be an object.");
Expand All @@ -82,7 +95,8 @@ private void validateObject(JsonNode object) throws ValidationException {
* "attributes": [
* ATTRIBUTE_NAME,
* ...
* ]
* ],
* "priority": INTEGER
* }
* </pre>
*
Expand All @@ -107,6 +121,9 @@ public void deserialize(JsonNode json) throws ValidationException {
case "attributes":
this.attributes(value);
break;
case "priority":
this.priority(value);
break;
default:
throw new ValidationException("'resolvers." + this.name + "." + name + "' is not a recognized field.");
}
Expand Down
79 changes: 74 additions & 5 deletions src/main/java/io/zentity/resolution/Job.java
Expand Up @@ -26,6 +26,7 @@

import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Map;
Expand Down Expand Up @@ -420,6 +421,24 @@ public static Map<String, Integer> countAttributesAcrossResolvers(Model model, L
return counts;
}

/**
* Group resolvers by their level of priority.
*
* @param model The entity model.
* @param resolvers The names of the resolvers to reference in the entity model.
* @return For each priority level, the names of the resolvers in that priority level.
*/
public static TreeMap<Integer, List<String>> groupResolversByPriority(Model model, List<String> resolvers) {
TreeMap<Integer, List<String>> resolverGroups = new TreeMap<>();
for (String resolverName : resolvers) {
Integer priority = model.resolvers().get(resolverName).priority();
if (!resolverGroups.containsKey(priority))
resolverGroups.put(priority, new ArrayList<>());
resolverGroups.get(priority).add(resolverName);
}
return resolverGroups;
}

/**
* Resets the variables that hold the state of the job, in case the same Job object is reused.
*/
Expand Down Expand Up @@ -601,11 +620,61 @@ else if (size == 1)
// Construct the resolvers clause.
String resolversClause = "{}";
TreeMap<String, TreeMap> resolversFilterTree = new TreeMap<>();
TreeMap<Integer, TreeMap<String, TreeMap>> resolversFilterTreeGrouped= new TreeMap<>();
if (!this.attributes.isEmpty()) {
Map<String, Integer> counts = countAttributesAcrossResolvers(this.input.model(), resolvers);
List<List<String>> resolversSorted = sortResolverAttributes(this.input.model(), resolvers, counts);
resolversFilterTree = makeResolversFilterTree(resolversSorted);
resolversClause = populateResolversFilterTree(this.input.model(), indexName, resolversFilterTree, this.attributes);

// Group the resolvers by their priority level.
TreeMap<Integer, List<String>> resolverGroups = groupResolversByPriority(this.input.model(), resolvers);

// Construct a clause for each priority level in descending order of priority.
List<Integer> priorities = new ArrayList<>(resolverGroups.keySet());
Collections.reverse(priorities);
for (int level = 0; level < priorities.size(); level++) {
Integer priority = priorities.get(level);
List<String> resolversGroup = resolverGroups.get(priority);
Map<String, Integer> counts = countAttributesAcrossResolvers(this.input.model(), resolversGroup);
List<List<String>> resolversSorted = sortResolverAttributes(this.input.model(), resolversGroup, counts);
resolversFilterTree = makeResolversFilterTree(resolversSorted);
resolversFilterTreeGrouped.put(level, resolversFilterTree);
resolversClause = populateResolversFilterTree(this.input.model(), indexName, resolversFilterTree, this.attributes);

// If there are multiple levels of priority, then each lower priority group of resolvers must ensure
// that every higher priority resolver either matches or does not exist.
List<String> parentResolversClauses = new ArrayList<>();
if (level > 0) {

// This is a lower priority group of resolvers.
// Every higher priority resolver either must match or must not exist.
for (int parentLevel = 0; parentLevel < level; parentLevel++) {
Integer parentPriority = priorities.get(parentLevel);
List<String> parentResolversGroup = resolverGroups.get(parentPriority);
List<String> parentResolverClauses = new ArrayList<>();
for (String parentResolverName : parentResolversGroup) {

// Construct a clause that checks if every attribute of the resolver does not exist.
List<String> attributeExistsClauses = new ArrayList<>();
for (String attributeName : this.input.model().resolvers().get(parentResolverName).attributes())
attributeExistsClauses.add("{\"exists\":{\"field\":\"" + attributeName + "\"}}");
String attributesExistsClause = "{\"bool\":{\"must_not\":[" + String.join(",", attributeExistsClauses) + "]}}";

// Construct a clause for the resolver.
List<String> parentResolverGroup = new ArrayList<>(Arrays.asList(parentResolverName));
Map<String, Integer> parentCounts = countAttributesAcrossResolvers(this.input.model(), parentResolverGroup);
List<List<String>> parentResolverSorted = sortResolverAttributes(this.input.model(), parentResolverGroup, parentCounts);
TreeMap<String, TreeMap> parentResolverFilterTree = makeResolversFilterTree(parentResolverSorted);
String parentResolverClause = populateResolversFilterTree(this.input.model(), indexName, parentResolverFilterTree, this.attributes);

// Construct a "should" clause for the above two clauses.
parentResolverClauses.add("{\"bool\":{\"should\":[" + attributesExistsClause + "," + parentResolverClause + "]}}");
}

// Construct a "filter" clause for every higher priority resolver clause.
parentResolversClauses.add("{\"bool\":{\"filter\":[" + String.join(",", parentResolverClauses) + "]}}");
}
}
if (parentResolversClauses.size() > 0)
resolversClause = "{\"bool\":{\"filter\":[" + resolversClause + "," + String.join(",", parentResolversClauses) + "]}}";
}
}

// Combine the ids clause and resolvers clause in a "should" clause if necessary.
Expand Down Expand Up @@ -661,7 +730,7 @@ else if (!idsClause.equals("{}"))
responseDataCopyObjHits.remove("hits");
}
String resolversListLogged = Json.ORDERED_MAPPER.writeValueAsString(resolvers);
String resolversFilterTreeLogged = Json.ORDERED_MAPPER.writeValueAsString(resolversFilterTree);
String resolversFilterTreeLogged = Json.ORDERED_MAPPER.writeValueAsString(resolversFilterTreeGrouped);
String resolversLogged = "{\"list\":" + resolversListLogged + ",\"tree\":" + resolversFilterTreeLogged + "}";
String searchLogged = "{\"request\":" + query + ",\"response\":" + responseDataCopyObj + "}";
String logged = "{\"_hop\":" + this.hop + ",\"_index\":\"" + indexName + "\",\"resolvers\":" + resolversLogged + ",\"search\":" + searchLogged + "}";
Expand Down

0 comments on commit a57958d

Please sign in to comment.