Skip to content

Commit

Permalink
First official attribute data types: string, number, boolean. Validat…
Browse files Browse the repository at this point in the history 
…ing data types of input attributes. Fixed data type conversions of returned _attributes field. Added tests.
  • Loading branch information
@davemoore-
davemoore- committed Apr 7, 2018
1 parent 88b4bc0 commit 19f7047
Show file tree
Hide file tree
Showing 13 changed files with 367 additions and 118 deletions.
89 changes: 86 additions & 3 deletions src/main/java/io/zentity/model/Attribute.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,14 @@

import java.io.IOException;
import java.util.*;
import java.util.regex.Pattern;

public class Attribute {

public static final Set<String> VALID_TYPES = new HashSet<>(
Arrays.asList("string", "long", "integer", "short", "byte", "double", "float", "boolean")
Arrays.asList("string", "number", "boolean")
);
private static final Pattern REGEX_EMPTY = Pattern.compile("^\\s*$");

private final String name;
private String type = "string";
Expand All @@ -27,6 +29,85 @@ public Attribute(String name, String json) throws ValidationException, IOExcepti
this.deserialize(json);
}

public static Boolean convertTypeBoolean(JsonNode value) {
if (value.isNull())
return null;
return value.booleanValue();
}

public static Number convertTypeNumber(JsonNode value) {
if (value.isNull())
return null;
else if (value.isIntegralNumber())
return value.bigIntegerValue();
else if (value.isFloatingPointNumber())
return value.doubleValue();
else
return value.numberValue();
}

public static String convertTypeString(JsonNode value) {
if (value.isNull())
return null;
return value.textValue();
}

public static Object convertType(String targetType, JsonNode value) throws ValidationException {
switch (targetType) {
case "boolean":
return convertTypeBoolean(value);
case "number":
return convertTypeNumber(value);
case "string":
return convertTypeString(value);
default:
throw new ValidationException("'" + targetType + " is not a recognized attribute data type.");
}
}

public static boolean isTypeBoolean(JsonNode value) {
return value.isBoolean();
}

public static boolean isTypeNumber(JsonNode value) {
return value.isNumber();
}

public static boolean isTypeString(JsonNode value) {
return value.isTextual();
}

public static void validateTypeBoolean(JsonNode value) throws ValidationException {
if (!isTypeBoolean(value) && !value.isNull())
throw new ValidationException("Expected 'boolean' attribute data type.");
}

public static void validateTypeNumber(JsonNode value) throws ValidationException {
if (!isTypeNumber(value) && !value.isNull())
throw new ValidationException("Expected 'number' attribute data type.");
}

public static void validateTypeString(JsonNode value) throws ValidationException {
if (!isTypeString(value) && !value.isNull())
throw new ValidationException("Expected 'string' attribute data type.");
}

public static void validateType(String expectedType, JsonNode value) throws ValidationException {
switch (expectedType) {
case "boolean":
validateTypeBoolean(value);
break;
case "number":
validateTypeNumber(value);
break;
case "string":
validateTypeString(value);
break;
default:
throw new ValidationException("'" + expectedType + " is not a recognized attribute data type.");
}
}

public String name() {
return this.name;
}
Expand All @@ -41,15 +122,17 @@ public void type(JsonNode value) throws ValidationException {
}

private void validateName(String value) throws ValidationException {
if (value.matches("^\\s*$"))
if (REGEX_EMPTY.matcher(value).matches())
throw new ValidationException("'attributes' has an attribute with empty name.");
}

private void validateType(JsonNode value) throws ValidationException {
if (!value.isTextual())
throw new ValidationException("'attributes." + this.name + ".type' must be a string.");
if (REGEX_EMPTY.matcher(value.textValue()).matches())
throw new ValidationException("'attributes." + this.name + ".type'' must not be empty.");
if (!VALID_TYPES.contains(value.textValue()))
throw new ValidationException("'attributes." + this.name + ".type' does not support the value '" + value.textValue() + "'.");
throw new ValidationException("'attributes." + this.name + ".type' has an unrecognized data type '" + value.textValue() + "'.");
}

private void validateObject(JsonNode object) throws ValidationException {
Expand Down
4 changes: 3 additions & 1 deletion src/main/java/io/zentity/model/Index.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,14 @@

import java.io.IOException;
import java.util.*;
import java.util.regex.Pattern;

public class Index {

public static final Set<String> REQUIRED_FIELDS = new HashSet<>(
Arrays.asList("fields")
);
private static final Pattern REGEX_EMPTY = Pattern.compile("^\\s*$");

private final String name;
private HashMap<String, IndexField> fields;
Expand Down Expand Up @@ -56,7 +58,7 @@ public void fields(JsonNode value) throws ValidationException {
}

private void validateName(String value) throws ValidationException {
if (value.matches("^\\s*$"))
if (REGEX_EMPTY.matcher(value).matches())
throw new ValidationException("'indices' has an index with an empty name.");
}

Expand Down
16 changes: 9 additions & 7 deletions src/main/java/io/zentity/model/IndexField.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,17 +5,19 @@

import java.io.IOException;
import java.util.*;
import java.util.regex.Pattern;

public class IndexField {

public static final Set<String> REQUIRED_FIELDS = new HashSet<>(
Arrays.asList("attribute")
);
private static final Pattern REGEX_EMPTY = Pattern.compile("^\\s*$");

private final String index;
private final String name;
private String attribute;
private String matcher = null;
private String matcher;

public IndexField(String index, String name, JsonNode json) throws ValidationException {
validateName(name);
Expand Down Expand Up @@ -54,25 +56,25 @@ public String matcher() {

public void matcher(JsonNode value) throws ValidationException {
validateMatcher(value);
this.matcher = value.isTextual() ? value.textValue() : null;
this.matcher = value.textValue();
}

private void validateName(String value) throws ValidationException {
if (value.matches("^\\s*$"))
if (REGEX_EMPTY.matcher(value).matches())
throw new ValidationException("'indices." + this.index + "' has a field with an empty name.");
}

private void validateAttribute(JsonNode value) throws ValidationException {
if (!value.isTextual())
throw new ValidationException("'indices." + this.index + "." + this.name + ".attribute' must be a string.");
if (value.textValue().matches("^\\s*$"))
if (REGEX_EMPTY.matcher(value.textValue()).matches())
throw new ValidationException("'indices." + this.index + "." + this.name + ".attribute' must not be empty.");
}

private void validateMatcher(JsonNode value) throws ValidationException {
if (!value.isTextual() && !value.isNull())
throw new ValidationException("'indices." + this.index + "." + this.name + ".matcher' must be a string or null.");
if (value.textValue().matches("^\\s*$"))
if (!value.isTextual())
throw new ValidationException("'indices." + this.index + "." + this.name + ".matcher' must be a string.");
if (REGEX_EMPTY.matcher(value.textValue()).matches())
throw new ValidationException("'indices." + this.index + "." + this.name + ".matcher' must not be empty.");
}

Expand Down
8 changes: 6 additions & 2 deletions src/main/java/io/zentity/model/Matcher.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@

import java.io.IOException;
import java.util.*;
import java.util.regex.Pattern;

public class Matcher {

Expand All @@ -15,6 +16,7 @@ public class Matcher {
public static final Set<String> VALID_TYPES = new HashSet<>(
Arrays.asList("value")
);
private static final Pattern REGEX_EMPTY = Pattern.compile("^\\s*$");
private static final ObjectMapper mapper = new ObjectMapper();

private final String name;
Expand Down Expand Up @@ -56,7 +58,7 @@ public void type(JsonNode value) throws ValidationException {
}

private void validateName(String value) throws ValidationException {
if (value.matches("^\\s*$"))
if (REGEX_EMPTY.matcher(value).matches())
throw new ValidationException("'matchers' field has a matcher with an empty name.");
}

Expand All @@ -70,8 +72,10 @@ private void validateClause(JsonNode value) throws ValidationException {
private void validateType(JsonNode value) throws ValidationException {
if (!value.isTextual())
throw new ValidationException("'matchers." + this.name + ".type' must be a string.");
if (REGEX_EMPTY.matcher(value.textValue()).matches())
throw new ValidationException("'attributes." + this.name + ".type'' must not be empty.");
if (!VALID_TYPES.contains(value.textValue()))
throw new ValidationException("'matchers." + this.name + ".type' does not support the value '" + value.textValue() + "'.");
throw new ValidationException("'matchers." + this.name + ".type' has an unrecognized data type '" + value.textValue() + "'.");
}

private void validateObject(JsonNode object) throws ValidationException {
Expand Down
8 changes: 5 additions & 3 deletions src/main/java/io/zentity/model/Resolver.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,14 @@

import java.io.IOException;
import java.util.*;
import java.util.regex.Pattern;

public class Resolver {

public static final Set<String> REQUIRED_FIELDS = new HashSet<>(
Arrays.asList("attributes")
);
private static final Pattern REGEX_EMPTY = Pattern.compile("^\\s*$");

private final String name;
private ArrayList<String> attributes = new ArrayList<>();
Expand Down Expand Up @@ -44,7 +46,7 @@ public void attributes(JsonNode value) throws ValidationException {
}

private void validateName(String value) throws ValidationException {
if (value.matches("^\\s*$"))
if (REGEX_EMPTY.matcher(value).matches())
throw new ValidationException("'resolvers' has a resolver with an empty name.");
}

Expand All @@ -57,8 +59,8 @@ private void validateAttributes(JsonNode value) throws ValidationException {
if (!attribute.isTextual())
throw new ValidationException("'resolvers." + this.name + ".attributes' must be an array of strings.");
String attributeName = attribute.textValue();
if (attributeName == null || attributeName.matches("^\\s*$"))
throw new ValidationException("'resolvers." + this.name + ".attributes' must be an array of strings.");
if (attributeName == null || REGEX_EMPTY.matcher(attributeName).matches())
throw new ValidationException("'resolvers." + this.name + ".attributes' must be an array of non-empty strings.");
}
}

Expand Down
2 changes: 1 addition & 1 deletion src/main/java/io/zentity/model/ValidationException.java
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
package io.zentity.model;

public class ValidationException extends Exception {
ValidationException(String message) {
public ValidationException(String message) {
super(message);
}
}
67 changes: 20 additions & 47 deletions src/main/java/io/zentity/resolution/Job.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,10 @@
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
import com.fasterxml.jackson.databind.node.ObjectNode;
import io.zentity.model.Attribute;
import io.zentity.model.Matcher;
import io.zentity.model.Model;
import io.zentity.model.ValidationException;
import org.elasticsearch.action.search.SearchAction;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
Expand Down Expand Up @@ -247,7 +249,7 @@ private SearchResponse search(String indexName, String query) throws IOException
*
* @throws IOException
*/
private void traverse() throws IOException {
private void traverse() throws IOException, ValidationException {

// Prepare to collect attributes from the results of these queries as the inputs to subsequent queries.
HashMap<String, HashSet<Object>> nextInputAttributes = new HashMap<>();
Expand Down Expand Up @@ -376,35 +378,21 @@ private void traverse() throws IOException {

// Gather attributes from the doc. Store them in the "_attributes" field of the doc,
// and include them in the attributes for subsequent queries.
TreeMap<String, Object> docAttributes = new TreeMap<>();
TreeMap<String, JsonNode> docAttributes = new TreeMap<>();
for (String indexFieldName : this.model.indices().get(indexName).fields().keySet()) {
String attributeName = this.model.indices().get(indexName).fields().get(indexFieldName).attribute();
String attributeType = this.model.attributes().get(attributeName).type();
// The index field name might not refer to the _source property.
// If it's not in the _source, remove the last part of the index field name from the dot notation.
// Index field names can reference multi-fields, which are not returned in the _source.
if (!nextInputAttributes.containsKey(attributeName))
nextInputAttributes.put(attributeName, new HashSet<>());
if (!doc.get("_source").has(indexFieldName))
indexFieldName = indexFieldName.split("\\.")[0];
if (doc.get("_source").has(indexFieldName)) {
Object value;
if (doc.get("_source").get(indexFieldName).isBoolean())
value = doc.get("_source").get(indexFieldName).booleanValue();
else if (doc.get("_source").get(indexFieldName).isDouble())
value = doc.get("_source").get(indexFieldName).doubleValue();
else if (doc.get("_source").get(indexFieldName).isFloat())
value = doc.get("_source").get(indexFieldName).floatValue();
else if (doc.get("_source").get(indexFieldName).isInt())
value = doc.get("_source").get(indexFieldName).intValue();
else if (doc.get("_source").get(indexFieldName).isLong())
value = doc.get("_source").get(indexFieldName).longValue();
else if (doc.get("_source").get(indexFieldName).isShort())
value = doc.get("_source").get(indexFieldName).shortValue();
else if (doc.get("_source").get(indexFieldName).isNull())
value = "";
else
value = doc.get("_source").get(indexFieldName).asText();
docAttributes.put(attributeName, value);
if (!nextInputAttributes.containsKey(attributeName))
nextInputAttributes.put(attributeName, new HashSet<>());
JsonNode valueNode = doc.get("_source").get(indexFieldName);
docAttributes.put(attributeName, valueNode);
Object value = Attribute.convertType(attributeType, valueNode);
nextInputAttributes.get(attributeName).add(value);
}
}
Expand All @@ -418,24 +406,9 @@ else if (doc.get("_source").get(indexFieldName).isNull())
docObjNode.remove("_source");
if (this.includeAttributes) {
ObjectNode docAttributesObjNode = docObjNode.putObject("_attributes");
for (String attribute : docAttributes.keySet()) {
Object value = docAttributes.get(attribute);
if (value.getClass() == Boolean.class)
docAttributesObjNode.put(attribute, (Boolean) value);
else if (value.getClass() == Double.class)
docAttributesObjNode.put(attribute, (Double) value);
else if (value.getClass() == Float.class)
docAttributesObjNode.put(attribute, (Float) value);
else if (value.getClass() == Integer.class)
docAttributesObjNode.put(attribute, (Integer) value);
else if (value.getClass() == Long.class)
docAttributesObjNode.put(attribute, (Long) value);
else if (value.getClass() == Short.class)
docAttributesObjNode.put(attribute, (Short) value);
else if (value.getClass() == null)
docAttributesObjNode.put(attribute, "");
else
docAttributesObjNode.put(attribute, (String) value);
for (String attributeName : docAttributes.keySet()) {
JsonNode values = docAttributes.get(attributeName);
docAttributesObjNode.set(attributeName, values);
}
}

Expand All @@ -451,12 +424,12 @@ else if (value.getClass() == null)
return;

// Update input attributes for the next queries.
for (String attribute : nextInputAttributes.keySet()) {
if (!this.inputAttributes.containsKey(attribute))
this.inputAttributes.put(attribute, new HashSet<>());
for (Object value : nextInputAttributes.get(attribute)) {
if (!this.inputAttributes.get(attribute).contains(value)) {
this.inputAttributes.get(attribute).add(value);
for (String attributeName : nextInputAttributes.keySet()) {
if (!this.inputAttributes.containsKey(attributeName))
this.inputAttributes.put(attributeName, new HashSet<>());
for (Object value : nextInputAttributes.get(attributeName)) {
if (!this.inputAttributes.get(attributeName).contains(value)) {
this.inputAttributes.get(attributeName).add(value);
newHits = true;
}
}
Expand All @@ -477,7 +450,7 @@ else if (value.getClass() == null)
* @return A JSON string to be returned as the body of the response to a client.
* @throws IOException
*/
public String run() throws IOException {
public String run() throws IOException, ValidationException {
try {

// Reset the state of the job if reusing this Job object.
Expand Down
Loading

0 comments on commit 19f7047

Please sign in to comment.