Skip to content

Commit

Permalink
ingest: dot_expander_processor, ignore_missing and prevent null
Browse files Browse the repository at this point in the history 
* support for ignore_missing, such that if the field does not exist in
source document the processor will be skipped

* fix bug such that if the location to expand to already exists as
nested objects, don't append a null value.
  • Loading branch information
@jakelandis
jakelandis committed Oct 30, 2018
1 parent 6abddd8 commit 650986d
Show file tree
Hide file tree
Showing 3 changed files with 58 additions and 18 deletions.
7 changes: 4 additions & 3 deletions docs/reference/ingest/ingest-node.asciidoc
View file
Expand Up @@ -1286,9 +1286,10 @@ Otherwise these <<accessing-data-in-pipelines,fields>> can't be accessed by any
.Dot Expand Options
[options="header"]
|======
| Name | Required | Default | Description
| `field` | yes | - | The field to expand into an object field
| `path` | no | - | The field that contains the field to expand. Only required if the field to expand is part another object field, because the `field` option can only understand leaf fields.
| Name | Required | Default | Description
| `field` | yes | - | The field to expand into an object field
| `path` | no | - | The field that contains the field to expand. Only required if the field to expand is part another object field, because the `field` option can only understand leaf fields.
| `ignore_missing` | no | `false` | If `true` and `field` does not exist or is `null`, the processor quietly exits without modifying the document
|======

[source,js]
Expand Down
14 changes: 10 additions & 4 deletions ...les/ingest-common/src/main/java/org/elasticsearch/ingest/common/DotExpanderProcessor.java
View file
Expand Up @@ -32,11 +32,13 @@ public final class DotExpanderProcessor extends AbstractProcessor {

private final String path;
private final String field;
private final boolean ignoreMissing;

DotExpanderProcessor(String tag, String path, String field) {
DotExpanderProcessor(String tag, String path, String field, boolean ignoreMissing) {
super(tag);
this.path = path;
this.field = field;
this.ignoreMissing = ignoreMissing;
}

@Override
Expand All @@ -54,8 +56,10 @@ public IngestDocument execute(IngestDocument ingestDocument) throws Exception {

if (ingestDocument.hasField(path)) {
Object value = map.remove(field);
ingestDocument.appendFieldValue(path, value);
} else {
if(value != null) {
ingestDocument.appendFieldValue(path, value);
}
} else if (ignoreMissing == false){
// check whether we actually can expand the field in question into an object field.
// part of the path may already exist and if part of it would be a value field (string, integer etc.)
// then we can't override it with an object field and we should fail with a good reason.
Expand Down Expand Up @@ -115,7 +119,9 @@ public Processor create(Map<String, Processor.Factory> processorFactories, Strin
}

String path = ConfigurationUtils.readOptionalStringProperty(TYPE, tag, config, "path");
return new DotExpanderProcessor(tag, path, field);
boolean ignoreMissing = ConfigurationUtils.readBooleanProperty(TYPE, tag, config, "ignore_missing", false);

return new DotExpanderProcessor(tag, path, field, ignoreMissing);
}
}
}
55 changes: 44 additions & 11 deletions ...ngest-common/src/test/java/org/elasticsearch/ingest/common/DotExpanderProcessorTests.java
View file
Expand Up @@ -37,15 +37,15 @@ public void testEscapeFields() throws Exception {
Map<String, Object> source = new HashMap<>();
source.put("foo.bar", "baz1");
IngestDocument document = new IngestDocument(source, Collections.emptyMap());
DotExpanderProcessor processor = new DotExpanderProcessor("_tag", null, "foo.bar");
DotExpanderProcessor processor = new DotExpanderProcessor("_tag", null, "foo.bar", false);
processor.execute(document);
assertThat(document.getFieldValue("foo", Map.class).size(), equalTo(1));
assertThat(document.getFieldValue("foo.bar", String.class), equalTo("baz1"));

source = new HashMap<>();
source.put("foo.bar.baz", "value");
document = new IngestDocument(source, Collections.emptyMap());
processor = new DotExpanderProcessor("_tag", null, "foo.bar.baz");
processor = new DotExpanderProcessor("_tag", null, "foo.bar.baz", false);
processor.execute(document);
assertThat(document.getFieldValue("foo", Map.class).size(), equalTo(1));
assertThat(document.getFieldValue("foo.bar", Map.class).size(), equalTo(1));
Expand All @@ -55,7 +55,7 @@ public void testEscapeFields() throws Exception {
source.put("foo.bar", "baz1");
source.put("foo", new HashMap<>(Collections.singletonMap("bar", "baz2")));
document = new IngestDocument(source, Collections.emptyMap());
processor = new DotExpanderProcessor("_tag", null, "foo.bar");
processor = new DotExpanderProcessor("_tag", null, "foo.bar", false);
processor.execute(document);
assertThat(document.getSourceAndMetadata().size(), equalTo(1));
assertThat(document.getFieldValue("foo.bar", List.class).size(), equalTo(2));
Expand All @@ -66,7 +66,7 @@ public void testEscapeFields() throws Exception {
source.put("foo.bar", "2");
source.put("foo", new HashMap<>(Collections.singletonMap("bar", 1)));
document = new IngestDocument(source, Collections.emptyMap());
processor = new DotExpanderProcessor("_tag", null, "foo.bar");
processor = new DotExpanderProcessor("_tag", null, "foo.bar", false);
processor.execute(document);
assertThat(document.getSourceAndMetadata().size(), equalTo(1));
assertThat(document.getFieldValue("foo.bar", List.class).size(), equalTo(2));
Expand All @@ -79,7 +79,7 @@ public void testEscapeFields_valueField() throws Exception {
source.put("foo.bar", "baz1");
source.put("foo", "baz2");
IngestDocument document1 = new IngestDocument(source, Collections.emptyMap());
Processor processor1 = new DotExpanderProcessor("_tag", null, "foo.bar");
Processor processor1 = new DotExpanderProcessor("_tag", null, "foo.bar", false);
// foo already exists and if a leaf field and therefor can't be replaced by a map field:
Exception e = expectThrows(IllegalArgumentException.class, () -> processor1.execute(document1));
assertThat(e.getMessage(), equalTo("cannot expend [foo.bar], because [foo] is not an object field, but a value field"));
Expand All @@ -90,7 +90,7 @@ public void testEscapeFields_valueField() throws Exception {
Processor processor = new RenameProcessor("_tag", new TestTemplateService.MockTemplateScript.Factory("foo"),
new TestTemplateService.MockTemplateScript.Factory("foo.bar"), false);
processor.execute(document);
processor = new DotExpanderProcessor("_tag", null, "foo.bar");
processor = new DotExpanderProcessor("_tag", null, "foo.bar", false);
processor.execute(document);
assertThat(document.getFieldValue("foo", Map.class).size(), equalTo(1));
assertThat(document.getFieldValue("foo.bar.0", String.class), equalTo("baz2"));
Expand All @@ -99,7 +99,7 @@ public void testEscapeFields_valueField() throws Exception {
source = new HashMap<>();
source.put("foo.bar", "baz1");
document = new IngestDocument(source, Collections.emptyMap());
processor = new DotExpanderProcessor("_tag", null, "foo.bar");
processor = new DotExpanderProcessor("_tag", null, "foo.bar", false);
processor.execute(document);
assertThat(document.getFieldValue("foo", Map.class).size(), equalTo(1));
assertThat(document.getFieldValue("foo.bar", String.class), equalTo("baz1"));
Expand All @@ -108,7 +108,7 @@ public void testEscapeFields_valueField() throws Exception {
source.put("foo.bar.baz", "baz1");
source.put("foo", new HashMap<>(Collections.singletonMap("bar", new HashMap<>())));
document = new IngestDocument(source, Collections.emptyMap());
processor = new DotExpanderProcessor("_tag", null, "foo.bar.baz");
processor = new DotExpanderProcessor("_tag", null, "foo.bar.baz", false);
processor.execute(document);
assertThat(document.getFieldValue("foo", Map.class).size(), equalTo(1));
assertThat(document.getFieldValue("foo.bar", Map.class).size(), equalTo(1));
Expand All @@ -118,7 +118,7 @@ public void testEscapeFields_valueField() throws Exception {
source.put("foo.bar.baz", "baz1");
source.put("foo", new HashMap<>(Collections.singletonMap("bar", "baz2")));
IngestDocument document2 = new IngestDocument(source, Collections.emptyMap());
Processor processor2 = new DotExpanderProcessor("_tag", null, "foo.bar.baz");
Processor processor2 = new DotExpanderProcessor("_tag", null, "foo.bar.baz", false);
e = expectThrows(IllegalArgumentException.class, () -> processor2.execute(document2));
assertThat(e.getMessage(), equalTo("cannot expend [foo.bar.baz], because [foo.bar] is not an object field, but a value field"));
}
Expand All @@ -127,7 +127,7 @@ public void testEscapeFields_path() throws Exception {
Map<String, Object> source = new HashMap<>();
source.put("foo", new HashMap<>(Collections.singletonMap("bar.baz", "value")));
IngestDocument document = new IngestDocument(source, Collections.emptyMap());
DotExpanderProcessor processor = new DotExpanderProcessor("_tag", "foo", "bar.baz");
DotExpanderProcessor processor = new DotExpanderProcessor("_tag", "foo", "bar.baz", false);
processor.execute(document);
assertThat(document.getFieldValue("foo", Map.class).size(), equalTo(1));
assertThat(document.getFieldValue("foo.bar", Map.class).size(), equalTo(1));
Expand All @@ -136,11 +136,44 @@ public void testEscapeFields_path() throws Exception {
source = new HashMap<>();
source.put("field", new HashMap<>(Collections.singletonMap("foo.bar.baz", "value")));
document = new IngestDocument(source, Collections.emptyMap());
processor = new DotExpanderProcessor("_tag", "field", "foo.bar.baz");
processor = new DotExpanderProcessor("_tag", "field", "foo.bar.baz", false);
processor.execute(document);
assertThat(document.getFieldValue("field.foo", Map.class).size(), equalTo(1));
assertThat(document.getFieldValue("field.foo.bar", Map.class).size(), equalTo(1));
assertThat(document.getFieldValue("field.foo.bar.baz", String.class), equalTo("value"));
}

public void testEscapeFields_ignoreMissing() throws Exception {
Map<String, Object> source = new HashMap<>();
source.put("foo.bar", "baz1");
IngestDocument document = new IngestDocument(source, Collections.emptyMap());
DotExpanderProcessor processor = new DotExpanderProcessor("_tag", null, "abc.def", true);
processor.execute(document);
assertFalse(document.hasField("foo.bar"));
assertFalse(document.hasField("abc.def"));

source = new HashMap<>();
source.put("foo.bar", "baz1");
document = new IngestDocument(source, Collections.emptyMap());
processor = new DotExpanderProcessor("_tag", null, "abc.def", false);
processor.execute(document);
assertFalse(document.hasField("foo.bar"));
//ignore_missing = false, so null field is created
assertTrue(document.hasField("abc.def"));
assertNull(document.getFieldValue("abc.def", Object.class));
}

public void testEscapeFields_doNotAddNull() throws Exception {
Map<String, Object> source = new HashMap<>();
Map<String, Object> inner = new HashMap<>();
inner.put("bar", "baz1");
source.put("foo", inner);
IngestDocument document = new IngestDocument(source, Collections.emptyMap());
DotExpanderProcessor processor = new DotExpanderProcessor("_tag", null, "foo.bar", false);
processor.execute(document);
assertTrue(document.hasField("foo.bar"));
//since the exact structure already exists don't append a null to existing structure
assertThat(document.getFieldValue("foo", Map.class).size(), equalTo(1));
assertThat(document.getFieldValue("foo.bar", String.class), equalTo("baz1"));
}
}

0 comments on commit 650986d

Please sign in to comment.