Skip to content
Permalink
Browse files

Bug fix: When a document retrieved from Elasticsearch lacks an index …

…field defined in the entity model, skip the field and attempt to extract the other fields. Previously, jobs were failing when index fields were missing from documents. But it's entirely valid for documents in Elasticsearch to lack fields that are defined in the index mapping.
  • Loading branch information...
davemoore- committed Sep 7, 2019
1 parent f4e629c commit cbaaa51538378611b76b25a64c2abbc6c17239da
Showing with 17 additions and 12 deletions.
  1. +7 −6 src/main/java/io/zentity/model/IndexField.java
  2. +10 −6 src/main/java/io/zentity/resolution/Job.java
@@ -1,5 +1,6 @@
package io.zentity.model;

import com.fasterxml.jackson.core.JsonPointer;
import com.fasterxml.jackson.databind.JsonNode;
import io.zentity.common.Json;
import io.zentity.common.Patterns;
@@ -19,8 +20,8 @@

private final String index;
private final String name;
private String path;
private String pathParent;
private JsonPointer path;
private JsonPointer pathParent;
private String attribute;
private String matcher;

@@ -48,11 +49,11 @@ public String name() {
return this.name;
}

public String path() {
public JsonPointer path() {
return this.path;
}

public String pathParent() {
public JsonPointer pathParent() {
return this.pathParent;
}

@@ -76,9 +77,9 @@ public void matcher(JsonNode value) throws ValidationException {

private void nameToPaths(String name) {
String[] parts = Patterns.PERIOD.split(name);
this.path = "/" + String.join("/", parts);
this.path = JsonPointer.compile("/" + String.join("/", parts));
if (parts.length > 1)
this.pathParent = "/" + String.join("/", Arrays.copyOf(parts, parts.length - 1));
this.pathParent = JsonPointer.compile("/" + String.join("/", Arrays.copyOf(parts, parts.length - 1)));
}

private void validateName(String value) throws ValidationException {
@@ -1,5 +1,6 @@
package io.zentity.resolution;

import com.fasterxml.jackson.core.JsonPointer;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ArrayNode;
@@ -1119,13 +1120,16 @@ else if (!resolversClause.isEmpty())
// The index field name might not refer to the _source property.
// If it's not in the _source, remove the last part of the index field name from the dot notation.
// Index field names can reference multi-fields, which are not returned in the _source.
String path = this.input.model().indices().get(indexName).fields().get(indexFieldName).path();
String pathParent = this.input.model().indices().get(indexName).fields().get(indexFieldName).pathParent();
// If the document does not contain a given index field, skip that field.
JsonPointer path = this.input.model().indices().get(indexName).fields().get(indexFieldName).path();
JsonPointer pathParent = this.input.model().indices().get(indexName).fields().get(indexFieldName).pathParent();
JsonNode valueNode = doc.get("_source").at(path);
if (valueNode.isMissingNode())
valueNode = doc.get("_source").at(pathParent);
if (valueNode.isMissingNode())
continue;
if (valueNode.isMissingNode()) {
if (pathParent != null)
valueNode = doc.get("_source").at(pathParent);
else
continue;
}
docAttributes.put(attributeName, valueNode);
Value value = Value.create(attributeType, valueNode);
nextInputAttributes.get(attributeName).values().add(value);

0 comments on commit cbaaa51

Please sign in to comment.
You can’t perform that action at this time.