Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
Added the "date" attribute type which requires a "format" param. Date… 
… values are queried by a given format

and returned as the same format regardless of how the date was formatted in the "_source" field.
  • Loading branch information
@davemoore-
davemoore- committed May 1, 2018
1 parent b2bdb09 commit 325b98c
Show file tree
Hide file tree
Showing 16 changed files with 358 additions and 80 deletions.
2 changes: 1 addition & 1 deletion src/main/java/io/zentity/common/Patterns.java
View file
Expand Up @@ -7,6 +7,6 @@ public class Patterns {
public static final Pattern EMPTY_STRING = Pattern.compile("^\\s*$");
public static final Pattern PERIOD = Pattern.compile("\\.");
public static final Pattern VARIABLE = Pattern.compile("\\{\\{\\s*([^\\s{}]+)\\s*}}");
public static final Pattern VARIABLE_PARAM = Pattern.compile("\\{\\{\\s*param\\.([^\\s{}]+)\\s*}}");
public static final Pattern VARIABLE_PARAMS = Pattern.compile("^params\\.(.+)");

}
4 changes: 2 additions & 2 deletions src/main/java/io/zentity/model/Attribute.java
View file
Expand Up @@ -16,7 +16,7 @@
public class Attribute {

public static final Set<String> VALID_TYPES = new TreeSet<>(
Arrays.asList("string", "number", "boolean")
Arrays.asList("boolean", "date", "number", "string")
);

private final String name;
Expand Down Expand Up @@ -130,7 +130,7 @@ public void deserialize(JsonNode json) throws ValidationException, JsonProcessin
Iterator<Map.Entry<String, JsonNode>> paramsNode = value.fields();
while (paramsNode.hasNext()) {
Map.Entry<String, JsonNode> paramNode = paramsNode.next();
String paramField = "params." + paramNode.getKey();
String paramField = paramNode.getKey();
JsonNode paramValue = paramNode.getValue();
if (paramValue.isObject() || paramValue.isArray())
this.params().put(paramField, Json.MAPPER.writeValueAsString(paramValue));
Expand Down
8 changes: 4 additions & 4 deletions src/main/java/io/zentity/model/Matcher.java
View file
Expand Up @@ -135,7 +135,7 @@ public void deserialize(JsonNode json) throws ValidationException, JsonProcessin
Iterator<Map.Entry<String, JsonNode>> paramsNode = value.fields();
while (paramsNode.hasNext()) {
Map.Entry<String, JsonNode> paramNode = paramsNode.next();
String paramField = "params." + paramNode.getKey();
String paramField = paramNode.getKey();
JsonNode paramValue = paramNode.getValue();
if (paramValue.isObject() || paramValue.isArray())
this.params().put(paramField, Json.MAPPER.writeValueAsString(paramValue));
Expand All @@ -146,11 +146,11 @@ else if (paramValue.isNull())
}

if (value.isObject() || value.isArray())
this.params().put("params." + name, Json.MAPPER.writeValueAsString(value));
this.params().put(name, Json.MAPPER.writeValueAsString(value));
else if (value.isNull())
this.params().put("params." + name, "null");
this.params().put(name, "null");
else
this.params().put("params." + name, value.asText());
this.params().put(name, value.asText());
break;
default:
throw new ValidationException("'matchers." + this.name + "." + name + "' is not a recognized field.");
Expand Down
151 changes: 125 additions & 26 deletions src/main/java/io/zentity/resolution/Job.java
View file
Expand Up @@ -4,6 +4,8 @@
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import io.zentity.common.Json;
import io.zentity.common.Patterns;
import io.zentity.model.Index;
import io.zentity.model.Matcher;
import io.zentity.model.Model;
import io.zentity.model.ValidationException;
Expand Down Expand Up @@ -70,6 +72,61 @@ public Job(NodeClient client) {
this.client = client;
}

public static String makeScriptFieldsClause(Input input, String indexName) throws ValidationException {
List<String> scriptFieldClauses = new ArrayList<>();

// Find any index fields that need to be included in the "script_fields" clause.
// Currently this includes any index field that is associated with a "date" attribute,
// which requires the "_source" value to be reformatted to a normalized format.
Index index = input.model().indices().get(indexName);
for (String attributeName : index.attributeIndexFieldsMap().keySet()) {
switch (input.model().attributes().get(attributeName).type()) {
case "date":

// Required params
String format;

// Make a "script" clause for each index field associated with this attribute.
for (String indexFieldName : index.attributeIndexFieldsMap().get(attributeName).keySet()) {
// Check if the required params are defined in the input attribute.
if (input.attributes().containsKey(attributeName) && input.attributes().get(attributeName).params().containsKey("format") && !input.attributes().get(attributeName).params().get("format").equals("null") && !Patterns.EMPTY_STRING.matcher(input.attributes().get(attributeName).params().get("format")).matches()) {
format = input.attributes().get(attributeName).params().get("format");
} else {
// Otherwise check if the required params are defined in the model attribute.
Map<String, String> params = input.model().attributes().get(attributeName).params();
if (params.containsKey("format") && !params.get("format").equals("null") && !Patterns.EMPTY_STRING.matcher(params.get("format")).matches()) {
format = params.get("format");
} else {
// Otherwise check if the required params are defined in the matcher associated with the index field.
String matcherName = index.attributeIndexFieldsMap().get(attributeName).get(indexFieldName).matcher();
params = input.model().matchers().get(matcherName).params();
if (params.containsKey("format") && !params.get("format").equals("null") && !Patterns.EMPTY_STRING.matcher(params.get("format")).matches()) {
format = params.get("format");
} else {
// If we've gotten this far, that means that the required params for this attribute type
// haven't been specified in any valid places.
throw new ValidationException("'attributes." + attributeName + "' is a 'date' which required a 'format' to be specified in the params.");
}
}
}

// Make the "script" clause
String scriptSource = "doc[params.field].value.toString(params.format)";
String scriptParams = "\"field\":\"" + indexFieldName + "\",\"format\":\"" + format + "\"";
String scriptFieldClause = "\"" + indexFieldName + "\":{\"script\":{\"lang\":\"painless\",\"source\":\"" + scriptSource + "\",\"params\":{" + scriptParams + "}}}";
scriptFieldClauses.add(scriptFieldClause);
}
break;

default:
break;
}
}
if (scriptFieldClauses.isEmpty())
return null;
return "\"script_fields\":{" + String.join(",", scriptFieldClauses) + "}";
}

/**
* Determine if a field of an index has a matcher associated with that field.
*
Expand Down Expand Up @@ -152,14 +209,18 @@ public static String populateMatcherClause(Matcher matcher, String indexFieldNam
matcherClause = pattern.matcher(matcherClause).replaceAll(value);
break;
default:
String paramValue;
if (attribute.params().containsKey(variable))
paramValue = attribute.params().get(variable);
else if (matcher.params().containsKey(variable))
paramValue = matcher.params().get(variable);
else
throw new ValidationException("'matchers." + matcher.name() + "' was given no value for '{{ " + variable + " }}'");
matcherClause = pattern.matcher(matcherClause).replaceAll(paramValue);
java.util.regex.Matcher m = Patterns.VARIABLE_PARAMS.matcher(variable);
if (m.find()) {
String var = m.group(1);
String paramValue;
if (attribute.params().containsKey(var))
paramValue = attribute.params().get(var);
else if (matcher.params().containsKey(var))
paramValue = matcher.params().get(var);
else
throw new ValidationException("'matchers." + matcher.name() + "' was given no value for '{{ " + variable + " }}'");
matcherClause = pattern.matcher(matcherClause).replaceAll(paramValue);
}
break;
}
}
Expand Down Expand Up @@ -502,6 +563,8 @@ private void traverse() throws IOException, ValidationException {
List<String> queryClauses = new ArrayList<>();
List<String> queryMustNotClauses = new ArrayList<>();
List<String> queryFilterClauses = new ArrayList<>();
List<String> topLevelClauses = new ArrayList<>();
topLevelClauses.add("\"_source\":true");

// Exclude docs by _id
Set<String> ids = this.docIds.get(indexName);
Expand Down Expand Up @@ -549,13 +612,24 @@ else if (size == 1)

// Construct the "query" clause.
if (!queryClauses.isEmpty())
queryClause = "{\"bool\":{" + String.join(",", queryClauses) + "}}";
queryClause = "\"query\":{\"bool\":{" + String.join(",", queryClauses) + "}}";
topLevelClauses.add(queryClause);

// Construct the final query.
// Construct the "script_fields" clause.
String scriptFieldsClause = makeScriptFieldsClause(this.input, indexName);
if (scriptFieldsClause != null)
topLevelClauses.add(scriptFieldsClause);

// Construct the "size" clause.
topLevelClauses.add("\"size\":" + this.maxDocsPerQuery);

// Construct the "profile" clause.
if (this.profile)
query = "{\"query\":" + queryClause + ",\"size\": " + this.maxDocsPerQuery + ",\"profile\":true}";
else
query = "{\"query\":" + queryClause + ",\"size\": " + this.maxDocsPerQuery + "}";
topLevelClauses.add("\"profile\":true");

// Construct the final query.
query = "{" + String.join(",", topLevelClauses) + "}";
System.out.println(query);

// Submit query to Elasticsearch.
SearchResponse response = this.search(indexName, query);
Expand Down Expand Up @@ -604,25 +678,50 @@ else if (size == 1)
String attributeType = this.input.model().attributes().get(attributeName).type();
if (!nextInputAttributes.containsKey(attributeName))
nextInputAttributes.put(attributeName, new Attribute(attributeName, attributeType));
// The index field name might not refer to the _source property.
// If it's not in the _source, remove the last part of the index field name from the dot notation.
// Index field names can reference multi-fields, which are not returned in the _source.
String path = this.input.model().indices().get(indexName).fields().get(indexFieldName).path();
String pathParent = this.input.model().indices().get(indexName).fields().get(indexFieldName).pathParent();
JsonNode valueNode = doc.get("_source").at(path);
if (valueNode.isMissingNode())
valueNode = doc.get("_source").at(pathParent);
if (valueNode.isMissingNode())
continue;
docAttributes.put(attributeName, valueNode);
Value value = Value.create(attributeType, valueNode);
nextInputAttributes.get(attributeName).values().add(value);

// Get the attribute value from the doc.
if (doc.has("fields") && doc.get("fields").has(indexFieldName)) {

// Get the attribute value from the "fields" field if it exists there.
// This would include 'date' attribute types, for example.
JsonNode valueNode = doc.get("fields").get(indexFieldName);
if (valueNode.size() > 1) {
docAttributes.put(attributeName, valueNode); // Return multiple values (as an array) in "_attributes"
for (JsonNode vNode : valueNode) {
Value value = Value.create(attributeType, vNode);
nextInputAttributes.get(attributeName).values().add(value);
}
} else {
JsonNode vNode = valueNode.get(0); // Return single value (not as an array) in "_attributes"
docAttributes.put(attributeName, vNode);
Value value = Value.create(attributeType, vNode);
nextInputAttributes.get(attributeName).values().add(value);
}

} else {

// Get the attribute value from the "_source" field.
// The index field name might not refer to the _source property.
// If it's not in the _source, remove the last part of the index field name from the dot notation.
// Index field names can reference multi-fields, which are not returned in the _source.
String path = this.input.model().indices().get(indexName).fields().get(indexFieldName).path();
String pathParent = this.input.model().indices().get(indexName).fields().get(indexFieldName).pathParent();
JsonNode valueNode = doc.get("_source").at(path);
if (valueNode.isMissingNode())
valueNode = doc.get("_source").at(pathParent);
if (valueNode.isMissingNode())
continue;
docAttributes.put(attributeName, valueNode);
Value value = Value.create(attributeType, valueNode);
nextInputAttributes.get(attributeName).values().add(value);
}
}

// Modify doc metadata.
if (this.includeHits) {
ObjectNode docObjNode = (ObjectNode) doc;
docObjNode.remove("_score");
docObjNode.remove("fields");
docObjNode.put("_hop", this.hop);
if (this.includeAttributes) {
ObjectNode docAttributesObjNode = docObjNode.putObject("_attributes");
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/io/zentity/resolution/input/Attribute.java
View file
Expand Up @@ -167,7 +167,7 @@ public void deserialize(JsonNode json) throws ValidationException, JsonProcessin
// Set any params that were specified in the input, with the values serialized as strings.
while (paramsNode.hasNext()) {
Map.Entry<String, JsonNode> paramNode = paramsNode.next();
String paramField = "params." + paramNode.getKey();
String paramField = paramNode.getKey();
JsonNode paramValue = paramNode.getValue();
if (paramValue.isObject() || paramValue.isArray())
this.params().put(paramField, Json.MAPPER.writeValueAsString(paramValue));
Expand Down
44 changes: 44 additions & 0 deletions src/main/java/io/zentity/resolution/input/Input.java
View file
Expand Up @@ -4,6 +4,7 @@
import com.fasterxml.jackson.databind.JsonNode;
import io.zentity.common.Json;
import io.zentity.common.Patterns;
import io.zentity.model.Index;
import io.zentity.model.Model;
import io.zentity.model.ValidationException;
import io.zentity.resolution.input.scope.Scope;
Expand All @@ -13,6 +14,7 @@
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import java.util.TreeSet;

public class Input {

Expand Down Expand Up @@ -269,6 +271,48 @@ public void deserialize(JsonNode json) throws ValidationException, IOException {
this.model = excludeResolvers(this.model, this.scope.exclude().resolvers());
}
}

// Validate that the attribute associated with each index field has any and all required params.
// For example, 'date' attributes require the 'format' field to be specified in the matcher params,
// the model attribute params, or the input attribute params so that the dates can be queried and returned
// in a normalized fashion. Currently this only applies to 'date' attribute types.
Set<String> paramsValidated = new TreeSet<>();
for (String indexName : this.model.indices().keySet()) {
Index index = this.model.indices().get(indexName);
for (String attributeName : index.attributeIndexFieldsMap().keySet()) {
if (paramsValidated.contains(attributeName))
continue;
if (!this.model.attributes().containsKey(attributeName))
continue;
switch (this.model.attributes().get(attributeName).type()) {
case "date":
// Check if the required params are defined in the input attribute.
Map<String, String> params = new TreeMap<>();
if (this.attributes.containsKey(attributeName))
params = this.attributes.get(attributeName).params();
if (!params.containsKey("format") || params.get("format").equals("null") || Patterns.EMPTY_STRING.matcher(params.get("format")).matches()) {
// Otherwise check if the required params are defined in the model attribute.
params = this.model.attributes().get(attributeName).params();
if (!params.containsKey("format") || params.get("format").equals("null") || Patterns.EMPTY_STRING.matcher(params.get("format")).matches()) {
// Otherwise check if the required params are defined in the matcher associated with the index field.
for (String indexFieldName : index.attributeIndexFieldsMap().get(attributeName).keySet()) {
String matcherName = index.attributeIndexFieldsMap().get(attributeName).get(indexFieldName).matcher();
params = this.model.matchers().get(matcherName).params();
if (!params.containsKey("format") || params.get("format").equals("null") || Patterns.EMPTY_STRING.matcher(params.get("format")).matches()) {
// If we've gotten this far, that means that the required params for this attribute type
// haven't been specified in any valid places.
throw new ValidationException("'attributes." + attributeName + "' is a 'date' which required a 'format' to be specified in the params.");
}
}
}
}
break;
default:
break;
}
paramsValidated.add(attributeName);
}
}
}

public void deserialize(String json) throws ValidationException, IOException {
Expand Down

0 comments on commit 325b98c

Please sign in to comment.