Skip to content
Permalink
Browse files

Introduced the concept of resolver priority. When a resolution job us…

…es resolvers with different priority levels, then the higher priority resolvers either must match the document or must not exist in the document.
  • Loading branch information...
davemoore- committed Jul 4, 2019
1 parent 6ad47c4 commit a57958dda8a1525a7d988890a21481d24212d8a8
Showing with 94 additions and 8 deletions.
  1. +1 −1 README.md
  2. +1 −1 pom.xml
  3. +18 −1 src/main/java/io/zentity/model/Resolver.java
  4. +74 −5 src/main/java/io/zentity/resolution/Job.java
@@ -31,7 +31,7 @@ Once you have installed Elasticsearch, you can install zentity from a remote URL

Example:

`elasticsearch-plugin install https://zentity.io/releases/zentity-1.0.3-elasticsearch-7.2.0.zip`
`elasticsearch-plugin install https://zentity.io/releases/zentity-1.1.0-elasticsearch-7.2.0.zip`

Read the [installation](https://zentity.io/docs/installation) docs for more details.

@@ -14,7 +14,7 @@
<zentity.author>Dave Moore</zentity.author>
<zentity.classname>org.elasticsearch.plugin.zentity.ZentityPlugin</zentity.classname>
<zentity.website>https://zentity.io</zentity.website>
<zentity.version>1.0.3</zentity.version>
<zentity.version>1.1.0</zentity.version>
<!-- dependency versions -->
<elasticsearch.version>7.2.0</elasticsearch.version>
<jackson.version>2.9.9</jackson.version>
@@ -19,6 +19,7 @@

private final String name;
private Set<String> attributes = new TreeSet<>();
private int priority = 0;

public Resolver(String name, JsonNode json) throws ValidationException {
validateName(name);
@@ -40,6 +41,8 @@ public String name() {
return this.attributes;
}

public int priority () { return this.priority; }

public void attributes(JsonNode value) throws ValidationException {
validateAttributes(value);
Set<String> attributes = new TreeSet<>();
@@ -48,6 +51,11 @@ public void attributes(JsonNode value) throws ValidationException {
this.attributes = attributes;
}

public void priority(JsonNode value) throws ValidationException {
validatePriority(value);
this.priority = value.asInt();
}

private void validateName(String value) throws ValidationException {
if (Patterns.EMPTY_STRING.matcher(value).matches())
throw new ValidationException("'resolvers' has a resolver with an empty name.");
@@ -67,6 +75,11 @@ private void validateAttributes(JsonNode value) throws ValidationException {
}
}

private void validatePriority(JsonNode value) throws ValidationException {
if (!value.isInt())
throw new ValidationException("'resolvers." + this.name + ".priority' must be an integer.");
}

private void validateObject(JsonNode object) throws ValidationException {
if (!object.isObject())
throw new ValidationException("'resolvers." + this.name + "' must be an object.");
@@ -82,7 +95,8 @@ private void validateObject(JsonNode object) throws ValidationException {
* "attributes": [
* ATTRIBUTE_NAME,
* ...
* ]
* ],
* "priority": INTEGER
* }
* </pre>
*
@@ -107,6 +121,9 @@ public void deserialize(JsonNode json) throws ValidationException {
case "attributes":
this.attributes(value);
break;
case "priority":
this.priority(value);
break;
default:
throw new ValidationException("'resolvers." + this.name + "." + name + "' is not a recognized field.");
}
@@ -26,6 +26,7 @@

import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Map;
@@ -420,6 +421,24 @@ else if (size == 1)
return counts;
}

/**
* Group resolvers by their level of priority.
*
* @param model The entity model.
* @param resolvers The names of the resolvers to reference in the entity model.
* @return For each priority level, the names of the resolvers in that priority level.
*/
public static TreeMap<Integer, List<String>> groupResolversByPriority(Model model, List<String> resolvers) {
TreeMap<Integer, List<String>> resolverGroups = new TreeMap<>();
for (String resolverName : resolvers) {
Integer priority = model.resolvers().get(resolverName).priority();
if (!resolverGroups.containsKey(priority))
resolverGroups.put(priority, new ArrayList<>());
resolverGroups.get(priority).add(resolverName);
}
return resolverGroups;
}

/**
* Resets the variables that hold the state of the job, in case the same Job object is reused.
*/
@@ -601,11 +620,61 @@ else if (size == 1)
// Construct the resolvers clause.
String resolversClause = "{}";
TreeMap<String, TreeMap> resolversFilterTree = new TreeMap<>();
TreeMap<Integer, TreeMap<String, TreeMap>> resolversFilterTreeGrouped= new TreeMap<>();
if (!this.attributes.isEmpty()) {
Map<String, Integer> counts = countAttributesAcrossResolvers(this.input.model(), resolvers);
List<List<String>> resolversSorted = sortResolverAttributes(this.input.model(), resolvers, counts);
resolversFilterTree = makeResolversFilterTree(resolversSorted);
resolversClause = populateResolversFilterTree(this.input.model(), indexName, resolversFilterTree, this.attributes);

// Group the resolvers by their priority level.
TreeMap<Integer, List<String>> resolverGroups = groupResolversByPriority(this.input.model(), resolvers);

// Construct a clause for each priority level in descending order of priority.
List<Integer> priorities = new ArrayList<>(resolverGroups.keySet());
Collections.reverse(priorities);
for (int level = 0; level < priorities.size(); level++) {
Integer priority = priorities.get(level);
List<String> resolversGroup = resolverGroups.get(priority);
Map<String, Integer> counts = countAttributesAcrossResolvers(this.input.model(), resolversGroup);
List<List<String>> resolversSorted = sortResolverAttributes(this.input.model(), resolversGroup, counts);
resolversFilterTree = makeResolversFilterTree(resolversSorted);
resolversFilterTreeGrouped.put(level, resolversFilterTree);
resolversClause = populateResolversFilterTree(this.input.model(), indexName, resolversFilterTree, this.attributes);

// If there are multiple levels of priority, then each lower priority group of resolvers must ensure
// that every higher priority resolver either matches or does not exist.
List<String> parentResolversClauses = new ArrayList<>();
if (level > 0) {

// This is a lower priority group of resolvers.
// Every higher priority resolver either must match or must not exist.
for (int parentLevel = 0; parentLevel < level; parentLevel++) {
Integer parentPriority = priorities.get(parentLevel);
List<String> parentResolversGroup = resolverGroups.get(parentPriority);
List<String> parentResolverClauses = new ArrayList<>();
for (String parentResolverName : parentResolversGroup) {

// Construct a clause that checks if every attribute of the resolver does not exist.
List<String> attributeExistsClauses = new ArrayList<>();
for (String attributeName : this.input.model().resolvers().get(parentResolverName).attributes())
attributeExistsClauses.add("{\"exists\":{\"field\":\"" + attributeName + "\"}}");
String attributesExistsClause = "{\"bool\":{\"must_not\":[" + String.join(",", attributeExistsClauses) + "]}}";

// Construct a clause for the resolver.
List<String> parentResolverGroup = new ArrayList<>(Arrays.asList(parentResolverName));
Map<String, Integer> parentCounts = countAttributesAcrossResolvers(this.input.model(), parentResolverGroup);
List<List<String>> parentResolverSorted = sortResolverAttributes(this.input.model(), parentResolverGroup, parentCounts);
TreeMap<String, TreeMap> parentResolverFilterTree = makeResolversFilterTree(parentResolverSorted);
String parentResolverClause = populateResolversFilterTree(this.input.model(), indexName, parentResolverFilterTree, this.attributes);

// Construct a "should" clause for the above two clauses.
parentResolverClauses.add("{\"bool\":{\"should\":[" + attributesExistsClause + "," + parentResolverClause + "]}}");
}

// Construct a "filter" clause for every higher priority resolver clause.
parentResolversClauses.add("{\"bool\":{\"filter\":[" + String.join(",", parentResolverClauses) + "]}}");
}
}
if (parentResolversClauses.size() > 0)
resolversClause = "{\"bool\":{\"filter\":[" + resolversClause + "," + String.join(",", parentResolversClauses) + "]}}";
}
}

// Combine the ids clause and resolvers clause in a "should" clause if necessary.
@@ -661,7 +730,7 @@ else if (!idsClause.equals("{}"))
responseDataCopyObjHits.remove("hits");
}
String resolversListLogged = Json.ORDERED_MAPPER.writeValueAsString(resolvers);
String resolversFilterTreeLogged = Json.ORDERED_MAPPER.writeValueAsString(resolversFilterTree);
String resolversFilterTreeLogged = Json.ORDERED_MAPPER.writeValueAsString(resolversFilterTreeGrouped);
String resolversLogged = "{\"list\":" + resolversListLogged + ",\"tree\":" + resolversFilterTreeLogged + "}";
String searchLogged = "{\"request\":" + query + ",\"response\":" + responseDataCopyObj + "}";
String logged = "{\"_hop\":" + this.hop + ",\"_index\":\"" + indexName + "\",\"resolvers\":" + resolversLogged + ",\"search\":" + searchLogged + "}";

0 comments on commit a57958d

Please sign in to comment.
You can’t perform that action at this time.