Skip to content

Commit 325b98c

Browse files
davemoore-davemoore-
committed
Added the "date" attribute type which requires a "format" param. Date values are queried by a given format
and returned as the same format regardless of how the date was formatted in the "_source" field.
1 parent b2bdb09 commit 325b98c

16 files changed

Lines changed: 358 additions & 80 deletions

File tree

‎src/main/java/io/zentity/common/Patterns.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,6 @@ public class Patterns {
77
public static final Pattern EMPTY_STRING = Pattern.compile("^\\s*$");
88
public static final Pattern PERIOD = Pattern.compile("\\.");
99
public static final Pattern VARIABLE = Pattern.compile("\\{\\{\\s*([^\\s{}]+)\\s*}}");
10-
public static final Pattern VARIABLE_PARAM = Pattern.compile("\\{\\{\\s*param\\.([^\\s{}]+)\\s*}}");
10+
public static final Pattern VARIABLE_PARAMS = Pattern.compile("^params\\.(.+)");
1111

1212
}

‎src/main/java/io/zentity/model/Attribute.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616
public class Attribute {
1717

1818
public static final Set<String> VALID_TYPES = new TreeSet<>(
19-
Arrays.asList("string", "number", "boolean")
19+
Arrays.asList("boolean", "date", "number", "string")
2020
);
2121

2222
private final String name;
@@ -130,7 +130,7 @@ public void deserialize(JsonNode json) throws ValidationException, JsonProcessin
130130
Iterator<Map.Entry<String, JsonNode>> paramsNode = value.fields();
131131
while (paramsNode.hasNext()) {
132132
Map.Entry<String, JsonNode> paramNode = paramsNode.next();
133-
String paramField = "params." + paramNode.getKey();
133+
String paramField = paramNode.getKey();
134134
JsonNode paramValue = paramNode.getValue();
135135
if (paramValue.isObject() || paramValue.isArray())
136136
this.params().put(paramField, Json.MAPPER.writeValueAsString(paramValue));

‎src/main/java/io/zentity/model/Matcher.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -135,7 +135,7 @@ public void deserialize(JsonNode json) throws ValidationException, JsonProcessin
135135
Iterator<Map.Entry<String, JsonNode>> paramsNode = value.fields();
136136
while (paramsNode.hasNext()) {
137137
Map.Entry<String, JsonNode> paramNode = paramsNode.next();
138-
String paramField = "params." + paramNode.getKey();
138+
String paramField = paramNode.getKey();
139139
JsonNode paramValue = paramNode.getValue();
140140
if (paramValue.isObject() || paramValue.isArray())
141141
this.params().put(paramField, Json.MAPPER.writeValueAsString(paramValue));
@@ -146,11 +146,11 @@ else if (paramValue.isNull())
146146
}
147147

148148
if (value.isObject() || value.isArray())
149-
this.params().put("params." + name, Json.MAPPER.writeValueAsString(value));
149+
this.params().put(name, Json.MAPPER.writeValueAsString(value));
150150
else if (value.isNull())
151-
this.params().put("params." + name, "null");
151+
this.params().put(name, "null");
152152
else
153-
this.params().put("params." + name, value.asText());
153+
this.params().put(name, value.asText());
154154
break;
155155
default:
156156
throw new ValidationException("'matchers." + this.name + "." + name + "' is not a recognized field.");

‎src/main/java/io/zentity/resolution/Job.java‎

Lines changed: 125 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@
44
import com.fasterxml.jackson.databind.JsonNode;
55
import com.fasterxml.jackson.databind.node.ObjectNode;
66
import io.zentity.common.Json;
7+
import io.zentity.common.Patterns;
8+
import io.zentity.model.Index;
79
import io.zentity.model.Matcher;
810
import io.zentity.model.Model;
911
import io.zentity.model.ValidationException;
@@ -70,6 +72,61 @@ public Job(NodeClient client) {
7072
this.client = client;
7173
}
7274

75+
public static String makeScriptFieldsClause(Input input, String indexName) throws ValidationException {
76+
List<String> scriptFieldClauses = new ArrayList<>();
77+
78+
// Find any index fields that need to be included in the "script_fields" clause.
79+
// Currently this includes any index field that is associated with a "date" attribute,
80+
// which requires the "_source" value to be reformatted to a normalized format.
81+
Index index = input.model().indices().get(indexName);
82+
for (String attributeName : index.attributeIndexFieldsMap().keySet()) {
83+
switch (input.model().attributes().get(attributeName).type()) {
84+
case "date":
85+
86+
// Required params
87+
String format;
88+
89+
// Make a "script" clause for each index field associated with this attribute.
90+
for (String indexFieldName : index.attributeIndexFieldsMap().get(attributeName).keySet()) {
91+
// Check if the required params are defined in the input attribute.
92+
if (input.attributes().containsKey(attributeName) && input.attributes().get(attributeName).params().containsKey("format") && !input.attributes().get(attributeName).params().get("format").equals("null") && !Patterns.EMPTY_STRING.matcher(input.attributes().get(attributeName).params().get("format")).matches()) {
93+
format = input.attributes().get(attributeName).params().get("format");
94+
} else {
95+
// Otherwise check if the required params are defined in the model attribute.
96+
Map<String, String> params = input.model().attributes().get(attributeName).params();
97+
if (params.containsKey("format") && !params.get("format").equals("null") && !Patterns.EMPTY_STRING.matcher(params.get("format")).matches()) {
98+
format = params.get("format");
99+
} else {
100+
// Otherwise check if the required params are defined in the matcher associated with the index field.
101+
String matcherName = index.attributeIndexFieldsMap().get(attributeName).get(indexFieldName).matcher();
102+
params = input.model().matchers().get(matcherName).params();
103+
if (params.containsKey("format") && !params.get("format").equals("null") && !Patterns.EMPTY_STRING.matcher(params.get("format")).matches()) {
104+
format = params.get("format");
105+
} else {
106+
// If we've gotten this far, that means that the required params for this attribute type
107+
// haven't been specified in any valid places.
108+
throw new ValidationException("'attributes." + attributeName + "' is a 'date' which required a 'format' to be specified in the params.");
109+
}
110+
}
111+
}
112+
113+
// Make the "script" clause
114+
String scriptSource = "doc[params.field].value.toString(params.format)";
115+
String scriptParams = "\"field\":\"" + indexFieldName + "\",\"format\":\"" + format + "\"";
116+
String scriptFieldClause = "\"" + indexFieldName + "\":{\"script\":{\"lang\":\"painless\",\"source\":\"" + scriptSource + "\",\"params\":{" + scriptParams + "}}}";
117+
scriptFieldClauses.add(scriptFieldClause);
118+
}
119+
break;
120+
121+
default:
122+
break;
123+
}
124+
}
125+
if (scriptFieldClauses.isEmpty())
126+
return null;
127+
return "\"script_fields\":{" + String.join(",", scriptFieldClauses) + "}";
128+
}
129+
73130
/**
74131
* Determine if a field of an index has a matcher associated with that field.
75132
*
@@ -152,14 +209,18 @@ public static String populateMatcherClause(Matcher matcher, String indexFieldNam
152209
matcherClause = pattern.matcher(matcherClause).replaceAll(value);
153210
break;
154211
default:
155-
String paramValue;
156-
if (attribute.params().containsKey(variable))
157-
paramValue = attribute.params().get(variable);
158-
else if (matcher.params().containsKey(variable))
159-
paramValue = matcher.params().get(variable);
160-
else
161-
throw new ValidationException("'matchers." + matcher.name() + "' was given no value for '{{ " + variable + " }}'");
162-
matcherClause = pattern.matcher(matcherClause).replaceAll(paramValue);
212+
java.util.regex.Matcher m = Patterns.VARIABLE_PARAMS.matcher(variable);
213+
if (m.find()) {
214+
String var = m.group(1);
215+
String paramValue;
216+
if (attribute.params().containsKey(var))
217+
paramValue = attribute.params().get(var);
218+
else if (matcher.params().containsKey(var))
219+
paramValue = matcher.params().get(var);
220+
else
221+
throw new ValidationException("'matchers." + matcher.name() + "' was given no value for '{{ " + variable + " }}'");
222+
matcherClause = pattern.matcher(matcherClause).replaceAll(paramValue);
223+
}
163224
break;
164225
}
165226
}
@@ -502,6 +563,8 @@ private void traverse() throws IOException, ValidationException {
502563
List<String> queryClauses = new ArrayList<>();
503564
List<String> queryMustNotClauses = new ArrayList<>();
504565
List<String> queryFilterClauses = new ArrayList<>();
566+
List<String> topLevelClauses = new ArrayList<>();
567+
topLevelClauses.add("\"_source\":true");
505568

506569
// Exclude docs by _id
507570
Set<String> ids = this.docIds.get(indexName);
@@ -549,13 +612,24 @@ else if (size == 1)
549612

550613
// Construct the "query" clause.
551614
if (!queryClauses.isEmpty())
552-
queryClause = "{\"bool\":{" + String.join(",", queryClauses) + "}}";
615+
queryClause = "\"query\":{\"bool\":{" + String.join(",", queryClauses) + "}}";
616+
topLevelClauses.add(queryClause);
553617

554-
// Construct the final query.
618+
// Construct the "script_fields" clause.
619+
String scriptFieldsClause = makeScriptFieldsClause(this.input, indexName);
620+
if (scriptFieldsClause != null)
621+
topLevelClauses.add(scriptFieldsClause);
622+
623+
// Construct the "size" clause.
624+
topLevelClauses.add("\"size\":" + this.maxDocsPerQuery);
625+
626+
// Construct the "profile" clause.
555627
if (this.profile)
556-
query = "{\"query\":" + queryClause + ",\"size\": " + this.maxDocsPerQuery + ",\"profile\":true}";
557-
else
558-
query = "{\"query\":" + queryClause + ",\"size\": " + this.maxDocsPerQuery + "}";
628+
topLevelClauses.add("\"profile\":true");
629+
630+
// Construct the final query.
631+
query = "{" + String.join(",", topLevelClauses) + "}";
632+
System.out.println(query);
559633

560634
// Submit query to Elasticsearch.
561635
SearchResponse response = this.search(indexName, query);
@@ -604,25 +678,50 @@ else if (size == 1)
604678
String attributeType = this.input.model().attributes().get(attributeName).type();
605679
if (!nextInputAttributes.containsKey(attributeName))
606680
nextInputAttributes.put(attributeName, new Attribute(attributeName, attributeType));
607-
// The index field name might not refer to the _source property.
608-
// If it's not in the _source, remove the last part of the index field name from the dot notation.
609-
// Index field names can reference multi-fields, which are not returned in the _source.
610-
String path = this.input.model().indices().get(indexName).fields().get(indexFieldName).path();
611-
String pathParent = this.input.model().indices().get(indexName).fields().get(indexFieldName).pathParent();
612-
JsonNode valueNode = doc.get("_source").at(path);
613-
if (valueNode.isMissingNode())
614-
valueNode = doc.get("_source").at(pathParent);
615-
if (valueNode.isMissingNode())
616-
continue;
617-
docAttributes.put(attributeName, valueNode);
618-
Value value = Value.create(attributeType, valueNode);
619-
nextInputAttributes.get(attributeName).values().add(value);
681+
682+
// Get the attribute value from the doc.
683+
if (doc.has("fields") && doc.get("fields").has(indexFieldName)) {
684+
685+
// Get the attribute value from the "fields" field if it exists there.
686+
// This would include 'date' attribute types, for example.
687+
JsonNode valueNode = doc.get("fields").get(indexFieldName);
688+
if (valueNode.size() > 1) {
689+
docAttributes.put(attributeName, valueNode); // Return multiple values (as an array) in "_attributes"
690+
for (JsonNode vNode : valueNode) {
691+
Value value = Value.create(attributeType, vNode);
692+
nextInputAttributes.get(attributeName).values().add(value);
693+
}
694+
} else {
695+
JsonNode vNode = valueNode.get(0); // Return single value (not as an array) in "_attributes"
696+
docAttributes.put(attributeName, vNode);
697+
Value value = Value.create(attributeType, vNode);
698+
nextInputAttributes.get(attributeName).values().add(value);
699+
}
700+
701+
} else {
702+
703+
// Get the attribute value from the "_source" field.
704+
// The index field name might not refer to the _source property.
705+
// If it's not in the _source, remove the last part of the index field name from the dot notation.
706+
// Index field names can reference multi-fields, which are not returned in the _source.
707+
String path = this.input.model().indices().get(indexName).fields().get(indexFieldName).path();
708+
String pathParent = this.input.model().indices().get(indexName).fields().get(indexFieldName).pathParent();
709+
JsonNode valueNode = doc.get("_source").at(path);
710+
if (valueNode.isMissingNode())
711+
valueNode = doc.get("_source").at(pathParent);
712+
if (valueNode.isMissingNode())
713+
continue;
714+
docAttributes.put(attributeName, valueNode);
715+
Value value = Value.create(attributeType, valueNode);
716+
nextInputAttributes.get(attributeName).values().add(value);
717+
}
620718
}
621719

622720
// Modify doc metadata.
623721
if (this.includeHits) {
624722
ObjectNode docObjNode = (ObjectNode) doc;
625723
docObjNode.remove("_score");
724+
docObjNode.remove("fields");
626725
docObjNode.put("_hop", this.hop);
627726
if (this.includeAttributes) {
628727
ObjectNode docAttributesObjNode = docObjNode.putObject("_attributes");

‎src/main/java/io/zentity/resolution/input/Attribute.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -167,7 +167,7 @@ public void deserialize(JsonNode json) throws ValidationException, JsonProcessin
167167
// Set any params that were specified in the input, with the values serialized as strings.
168168
while (paramsNode.hasNext()) {
169169
Map.Entry<String, JsonNode> paramNode = paramsNode.next();
170-
String paramField = "params." + paramNode.getKey();
170+
String paramField = paramNode.getKey();
171171
JsonNode paramValue = paramNode.getValue();
172172
if (paramValue.isObject() || paramValue.isArray())
173173
this.params().put(paramField, Json.MAPPER.writeValueAsString(paramValue));

‎src/main/java/io/zentity/resolution/input/Input.java‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
import com.fasterxml.jackson.databind.JsonNode;
55
import io.zentity.common.Json;
66
import io.zentity.common.Patterns;
7+
import io.zentity.model.Index;
78
import io.zentity.model.Model;
89
import io.zentity.model.ValidationException;
910
import io.zentity.resolution.input.scope.Scope;
@@ -13,6 +14,7 @@
1314
import java.util.Map;
1415
import java.util.Set;
1516
import java.util.TreeMap;
17+
import java.util.TreeSet;
1618

1719
public class Input {
1820

@@ -269,6 +271,48 @@ public void deserialize(JsonNode json) throws ValidationException, IOException {
269271
this.model = excludeResolvers(this.model, this.scope.exclude().resolvers());
270272
}
271273
}
274+
275+
// Validate that the attribute associated with each index field has any and all required params.
276+
// For example, 'date' attributes require the 'format' field to be specified in the matcher params,
277+
// the model attribute params, or the input attribute params so that the dates can be queried and returned
278+
// in a normalized fashion. Currently this only applies to 'date' attribute types.
279+
Set<String> paramsValidated = new TreeSet<>();
280+
for (String indexName : this.model.indices().keySet()) {
281+
Index index = this.model.indices().get(indexName);
282+
for (String attributeName : index.attributeIndexFieldsMap().keySet()) {
283+
if (paramsValidated.contains(attributeName))
284+
continue;
285+
if (!this.model.attributes().containsKey(attributeName))
286+
continue;
287+
switch (this.model.attributes().get(attributeName).type()) {
288+
case "date":
289+
// Check if the required params are defined in the input attribute.
290+
Map<String, String> params = new TreeMap<>();
291+
if (this.attributes.containsKey(attributeName))
292+
params = this.attributes.get(attributeName).params();
293+
if (!params.containsKey("format") || params.get("format").equals("null") || Patterns.EMPTY_STRING.matcher(params.get("format")).matches()) {
294+
// Otherwise check if the required params are defined in the model attribute.
295+
params = this.model.attributes().get(attributeName).params();
296+
if (!params.containsKey("format") || params.get("format").equals("null") || Patterns.EMPTY_STRING.matcher(params.get("format")).matches()) {
297+
// Otherwise check if the required params are defined in the matcher associated with the index field.
298+
for (String indexFieldName : index.attributeIndexFieldsMap().get(attributeName).keySet()) {
299+
String matcherName = index.attributeIndexFieldsMap().get(attributeName).get(indexFieldName).matcher();
300+
params = this.model.matchers().get(matcherName).params();
301+
if (!params.containsKey("format") || params.get("format").equals("null") || Patterns.EMPTY_STRING.matcher(params.get("format")).matches()) {
302+
// If we've gotten this far, that means that the required params for this attribute type
303+
// haven't been specified in any valid places.
304+
throw new ValidationException("'attributes." + attributeName + "' is a 'date' which required a 'format' to be specified in the params.");
305+
}
306+
}
307+
}
308+
}
309+
break;
310+
default:
311+
break;
312+
}
313+
paramsValidated.add(attributeName);
314+
}
315+
}
272316
}
273317

274318
public void deserialize(String json) throws ValidationException, IOException {

0 commit comments

Comments
 (0)