|
|
@@ -4,6 +4,8 @@ |
|
|
import com.fasterxml.jackson.databind.JsonNode;
|
|
|
import com.fasterxml.jackson.databind.node.ObjectNode;
|
|
|
import io.zentity.common.Json;
|
|
|
import io.zentity.common.Patterns;
|
|
|
import io.zentity.model.Index;
|
|
|
import io.zentity.model.Matcher;
|
|
|
import io.zentity.model.Model;
|
|
|
import io.zentity.model.ValidationException;
|
|
|
@@ -70,6 +72,61 @@ public Job(NodeClient client) { |
|
|
this.client = client;
|
|
|
}
|
|
|
|
|
|
public static String makeScriptFieldsClause(Input input, String indexName) throws ValidationException {
|
|
|
List<String> scriptFieldClauses = new ArrayList<>();
|
|
|
|
|
|
// Find any index fields that need to be included in the "script_fields" clause.
|
|
|
// Currently this includes any index field that is associated with a "date" attribute,
|
|
|
// which requires the "_source" value to be reformatted to a normalized format.
|
|
|
Index index = input.model().indices().get(indexName);
|
|
|
for (String attributeName : index.attributeIndexFieldsMap().keySet()) {
|
|
|
switch (input.model().attributes().get(attributeName).type()) {
|
|
|
case "date":
|
|
|
|
|
|
// Required params
|
|
|
String format;
|
|
|
|
|
|
// Make a "script" clause for each index field associated with this attribute.
|
|
|
for (String indexFieldName : index.attributeIndexFieldsMap().get(attributeName).keySet()) {
|
|
|
// Check if the required params are defined in the input attribute.
|
|
|
if (input.attributes().containsKey(attributeName) && input.attributes().get(attributeName).params().containsKey("format") && !input.attributes().get(attributeName).params().get("format").equals("null") && !Patterns.EMPTY_STRING.matcher(input.attributes().get(attributeName).params().get("format")).matches()) {
|
|
|
format = input.attributes().get(attributeName).params().get("format");
|
|
|
} else {
|
|
|
// Otherwise check if the required params are defined in the model attribute.
|
|
|
Map<String, String> params = input.model().attributes().get(attributeName).params();
|
|
|
if (params.containsKey("format") && !params.get("format").equals("null") && !Patterns.EMPTY_STRING.matcher(params.get("format")).matches()) {
|
|
|
format = params.get("format");
|
|
|
} else {
|
|
|
// Otherwise check if the required params are defined in the matcher associated with the index field.
|
|
|
String matcherName = index.attributeIndexFieldsMap().get(attributeName).get(indexFieldName).matcher();
|
|
|
params = input.model().matchers().get(matcherName).params();
|
|
|
if (params.containsKey("format") && !params.get("format").equals("null") && !Patterns.EMPTY_STRING.matcher(params.get("format")).matches()) {
|
|
|
format = params.get("format");
|
|
|
} else {
|
|
|
// If we've gotten this far, that means that the required params for this attribute type
|
|
|
// haven't been specified in any valid places.
|
|
|
throw new ValidationException("'attributes." + attributeName + "' is a 'date' which required a 'format' to be specified in the params.");
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
|
|
|
// Make the "script" clause
|
|
|
String scriptSource = "doc[params.field].value.toString(params.format)";
|
|
|
String scriptParams = "\"field\":\"" + indexFieldName + "\",\"format\":\"" + format + "\"";
|
|
|
String scriptFieldClause = "\"" + indexFieldName + "\":{\"script\":{\"lang\":\"painless\",\"source\":\"" + scriptSource + "\",\"params\":{" + scriptParams + "}}}";
|
|
|
scriptFieldClauses.add(scriptFieldClause);
|
|
|
}
|
|
|
break;
|
|
|
|
|
|
default:
|
|
|
break;
|
|
|
}
|
|
|
}
|
|
|
if (scriptFieldClauses.isEmpty())
|
|
|
return null;
|
|
|
return "\"script_fields\":{" + String.join(",", scriptFieldClauses) + "}";
|
|
|
}
|
|
|
|
|
|
/**
|
|
|
* Determine if a field of an index has a matcher associated with that field.
|
|
|
*
|
|
|
@@ -152,14 +209,18 @@ public static String populateMatcherClause(Matcher matcher, String indexFieldNam |
|
|
matcherClause = pattern.matcher(matcherClause).replaceAll(value);
|
|
|
break;
|
|
|
default:
|
|
|
String paramValue;
|
|
|
if (attribute.params().containsKey(variable))
|
|
|
paramValue = attribute.params().get(variable);
|
|
|
else if (matcher.params().containsKey(variable))
|
|
|
paramValue = matcher.params().get(variable);
|
|
|
else
|
|
|
throw new ValidationException("'matchers." + matcher.name() + "' was given no value for '{{ " + variable + " }}'");
|
|
|
matcherClause = pattern.matcher(matcherClause).replaceAll(paramValue);
|
|
|
java.util.regex.Matcher m = Patterns.VARIABLE_PARAMS.matcher(variable);
|
|
|
if (m.find()) {
|
|
|
String var = m.group(1);
|
|
|
String paramValue;
|
|
|
if (attribute.params().containsKey(var))
|
|
|
paramValue = attribute.params().get(var);
|
|
|
else if (matcher.params().containsKey(var))
|
|
|
paramValue = matcher.params().get(var);
|
|
|
else
|
|
|
throw new ValidationException("'matchers." + matcher.name() + "' was given no value for '{{ " + variable + " }}'");
|
|
|
matcherClause = pattern.matcher(matcherClause).replaceAll(paramValue);
|
|
|
}
|
|
|
break;
|
|
|
}
|
|
|
}
|
|
|
@@ -502,6 +563,8 @@ private void traverse() throws IOException, ValidationException { |
|
|
List<String> queryClauses = new ArrayList<>();
|
|
|
List<String> queryMustNotClauses = new ArrayList<>();
|
|
|
List<String> queryFilterClauses = new ArrayList<>();
|
|
|
List<String> topLevelClauses = new ArrayList<>();
|
|
|
topLevelClauses.add("\"_source\":true");
|
|
|
|
|
|
// Exclude docs by _id
|
|
|
Set<String> ids = this.docIds.get(indexName);
|
|
|
@@ -549,13 +612,24 @@ else if (size == 1) |
|
|
|
|
|
// Construct the "query" clause.
|
|
|
if (!queryClauses.isEmpty())
|
|
|
queryClause = "{\"bool\":{" + String.join(",", queryClauses) + "}}";
|
|
|
queryClause = "\"query\":{\"bool\":{" + String.join(",", queryClauses) + "}}";
|
|
|
topLevelClauses.add(queryClause);
|
|
|
|
|
|
// Construct the final query.
|
|
|
// Construct the "script_fields" clause.
|
|
|
String scriptFieldsClause = makeScriptFieldsClause(this.input, indexName);
|
|
|
if (scriptFieldsClause != null)
|
|
|
topLevelClauses.add(scriptFieldsClause);
|
|
|
|
|
|
// Construct the "size" clause.
|
|
|
topLevelClauses.add("\"size\":" + this.maxDocsPerQuery);
|
|
|
|
|
|
// Construct the "profile" clause.
|
|
|
if (this.profile)
|
|
|
query = "{\"query\":" + queryClause + ",\"size\": " + this.maxDocsPerQuery + ",\"profile\":true}";
|
|
|
else
|
|
|
query = "{\"query\":" + queryClause + ",\"size\": " + this.maxDocsPerQuery + "}";
|
|
|
topLevelClauses.add("\"profile\":true");
|
|
|
|
|
|
// Construct the final query.
|
|
|
query = "{" + String.join(",", topLevelClauses) + "}";
|
|
|
System.out.println(query);
|
|
|
|
|
|
// Submit query to Elasticsearch.
|
|
|
SearchResponse response = this.search(indexName, query);
|
|
|
@@ -604,25 +678,50 @@ else if (size == 1) |
|
|
String attributeType = this.input.model().attributes().get(attributeName).type();
|
|
|
if (!nextInputAttributes.containsKey(attributeName))
|
|
|
nextInputAttributes.put(attributeName, new Attribute(attributeName, attributeType));
|
|
|
// The index field name might not refer to the _source property.
|
|
|
// If it's not in the _source, remove the last part of the index field name from the dot notation.
|
|
|
// Index field names can reference multi-fields, which are not returned in the _source.
|
|
|
String path = this.input.model().indices().get(indexName).fields().get(indexFieldName).path();
|
|
|
String pathParent = this.input.model().indices().get(indexName).fields().get(indexFieldName).pathParent();
|
|
|
JsonNode valueNode = doc.get("_source").at(path);
|
|
|
if (valueNode.isMissingNode())
|
|
|
valueNode = doc.get("_source").at(pathParent);
|
|
|
if (valueNode.isMissingNode())
|
|
|
continue;
|
|
|
docAttributes.put(attributeName, valueNode);
|
|
|
Value value = Value.create(attributeType, valueNode);
|
|
|
nextInputAttributes.get(attributeName).values().add(value);
|
|
|
|
|
|
// Get the attribute value from the doc.
|
|
|
if (doc.has("fields") && doc.get("fields").has(indexFieldName)) {
|
|
|
|
|
|
// Get the attribute value from the "fields" field if it exists there.
|
|
|
// This would include 'date' attribute types, for example.
|
|
|
JsonNode valueNode = doc.get("fields").get(indexFieldName);
|
|
|
if (valueNode.size() > 1) {
|
|
|
docAttributes.put(attributeName, valueNode); // Return multiple values (as an array) in "_attributes"
|
|
|
for (JsonNode vNode : valueNode) {
|
|
|
Value value = Value.create(attributeType, vNode);
|
|
|
nextInputAttributes.get(attributeName).values().add(value);
|
|
|
}
|
|
|
} else {
|
|
|
JsonNode vNode = valueNode.get(0); // Return single value (not as an array) in "_attributes"
|
|
|
docAttributes.put(attributeName, vNode);
|
|
|
Value value = Value.create(attributeType, vNode);
|
|
|
nextInputAttributes.get(attributeName).values().add(value);
|
|
|
}
|
|
|
|
|
|
} else {
|
|
|
|
|
|
// Get the attribute value from the "_source" field.
|
|
|
// The index field name might not refer to the _source property.
|
|
|
// If it's not in the _source, remove the last part of the index field name from the dot notation.
|
|
|
// Index field names can reference multi-fields, which are not returned in the _source.
|
|
|
String path = this.input.model().indices().get(indexName).fields().get(indexFieldName).path();
|
|
|
String pathParent = this.input.model().indices().get(indexName).fields().get(indexFieldName).pathParent();
|
|
|
JsonNode valueNode = doc.get("_source").at(path);
|
|
|
if (valueNode.isMissingNode())
|
|
|
valueNode = doc.get("_source").at(pathParent);
|
|
|
if (valueNode.isMissingNode())
|
|
|
continue;
|
|
|
docAttributes.put(attributeName, valueNode);
|
|
|
Value value = Value.create(attributeType, valueNode);
|
|
|
nextInputAttributes.get(attributeName).values().add(value);
|
|
|
}
|
|
|
}
|
|
|
|
|
|
// Modify doc metadata.
|
|
|
if (this.includeHits) {
|
|
|
ObjectNode docObjNode = (ObjectNode) doc;
|
|
|
docObjNode.remove("_score");
|
|
|
docObjNode.remove("fields");
|
|
|
docObjNode.put("_hop", this.hop);
|
|
|
if (this.includeAttributes) {
|
|
|
ObjectNode docAttributesObjNode = docObjNode.putObject("_attributes");
|
|
|
|
0 comments on commit
325b98c