From 4801bffb45649501f4c51f197de4dc8a10953e08 Mon Sep 17 00:00:00 2001 From: Nick Allen Date: Wed, 4 Apr 2018 15:29:26 -0400 Subject: [PATCH 1/2] METRON-1511 Unable to Serialize Profiler Configuration --- .../configuration/profiler/ProfileConfig.java | 36 +++++- .../profiler/ProfileResultExpressions.java | 4 +- .../profiler/ProfileTriageExpressions.java | 8 ++ .../profiler/ProfilerConfig.java | 47 ++++++++ .../profiler/ProfileConfigTest.java | 102 +++++++++++++--- .../profiler/ProfilerConfigTest.java | 109 ++++++++++++++++-- 6 files changed, 276 insertions(+), 30 deletions(-) diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileConfig.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileConfig.java index 6205fbf377..7e2ec6c228 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileConfig.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileConfig.java @@ -18,10 +18,12 @@ package org.apache.metron.common.configuration.profiler; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.core.JsonProcessingException; +import org.apache.metron.common.utils.JSONUtils; +import java.io.IOException; import java.io.Serializable; import java.util.ArrayList; -import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -266,4 +268,36 @@ public String toString() { ", expires=" + expires + '}'; } + + /** + * Deserialize a {@link ProfileConfig}. + * + * @param bytes Raw bytes containing a UTF-8 JSON String. + * @return The Profile definition. + * @throws IOException + */ + public static ProfileConfig fromBytes(byte[] bytes) throws IOException { + return JSONUtils.INSTANCE.load(new String(bytes), ProfileConfig.class); + } + + /** + * Deserialize a {@link ProfileConfig}. + * + * @param json A String containing JSON. + * @return The Profile definition. + * @throws IOException + */ + public static ProfileConfig fromJSON(String json) throws IOException { + return JSONUtils.INSTANCE.load(json, ProfileConfig.class); + } + + /** + * Serialize the profile definition to a JSON string. + * + * @return The Profiler configuration serialized as a JSON string. + * @throws JsonProcessingException + */ + public String toJSON() throws JsonProcessingException { + return JSONUtils.INSTANCE.toJSON(this, true); + } } diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileResultExpressions.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileResultExpressions.java index 82af223d86..5bcec7227d 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileResultExpressions.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileResultExpressions.java @@ -18,7 +18,7 @@ package org.apache.metron.common.configuration.profiler; import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonValue; /** * A Stellar expression that is executed to produce a single @@ -26,7 +26,6 @@ */ public class ProfileResultExpressions { - @JsonIgnore private String expression; @JsonCreator @@ -34,6 +33,7 @@ public ProfileResultExpressions(String expression) { this.expression = expression; } + @JsonValue public String getExpression() { return expression; } diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileTriageExpressions.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileTriageExpressions.java index fbe1706b4d..da02cb2d8c 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileTriageExpressions.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileTriageExpressions.java @@ -17,6 +17,8 @@ */ package org.apache.metron.common.configuration.profiler; +import com.fasterxml.jackson.annotation.JsonAnyGetter; +import com.fasterxml.jackson.annotation.JsonAnySetter; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; @@ -61,10 +63,16 @@ public String getExpression(String name) { return expressions.get(name); } + @JsonAnyGetter public Map getExpressions() { return expressions; } + @JsonAnySetter + public void setExpressions(Map expressions) { + this.expressions = expressions; + } + @Override public String toString() { return "ProfileTriageExpressions{" + diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfilerConfig.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfilerConfig.java index 0bdb7e2ffc..451c86bda9 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfilerConfig.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfilerConfig.java @@ -17,6 +17,14 @@ */ package org.apache.metron.common.configuration.profiler; +import com.fasterxml.jackson.annotation.JsonGetter; +import com.fasterxml.jackson.annotation.JsonSetter; +import com.fasterxml.jackson.core.JsonProcessingException; +import org.apache.metron.common.utils.JSONUtils; +import org.codehaus.jackson.map.annotate.JsonSerialize; +import org.codehaus.jackson.map.annotate.JsonSerialize.Inclusion; + +import java.io.IOException; import java.io.Serializable; import java.util.ArrayList; import java.util.List; @@ -25,6 +33,7 @@ /** * The configuration object for the Profiler, which may contain many Profile definitions. */ +@JsonSerialize(include=Inclusion.NON_NULL) public class ProfilerConfig implements Serializable { /** @@ -59,10 +68,16 @@ public ProfilerConfig withProfile(ProfileConfig profileConfig) { return this; } + @JsonGetter("timestampField") + public String getTimestampFieldForJson() { + return timestampField.orElse(null); + } + public Optional getTimestampField() { return timestampField; } + @JsonSetter("timestampField") public void setTimestampField(String timestampField) { this.timestampField = Optional.of(timestampField); } @@ -99,4 +114,36 @@ public int hashCode() { result = 31 * result + (timestampField != null ? timestampField.hashCode() : 0); return result; } + + /** + * Deserialize a {@link ProfilerConfig}. + * + * @param bytes Raw bytes containing a UTF-8 JSON String. + * @return The Profiler configuration. + * @throws IOException + */ + public static ProfilerConfig fromBytes(byte[] bytes) throws IOException { + return JSONUtils.INSTANCE.load(new String(bytes), ProfilerConfig.class); + } + + /** + * Deserialize a {@link ProfilerConfig}. + * + * @param json A String containing JSON. + * @return The Profiler configuration. + * @throws IOException + */ + public static ProfilerConfig fromJSON(String json) throws IOException { + return JSONUtils.INSTANCE.load(json, ProfilerConfig.class); + } + + /** + * Serialize a {@link ProfilerConfig} to a JSON string. + * + * @return The Profiler configuration serialized as a JSON string. + * @throws JsonProcessingException + */ + public String toJSON() throws JsonProcessingException { + return JSONUtils.INSTANCE.toJSON(this, true); + } } diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/profiler/ProfileConfigTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/profiler/ProfileConfigTest.java index e178ee02e1..87dbbc4330 100644 --- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/profiler/ProfileConfigTest.java +++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/profiler/ProfileConfigTest.java @@ -21,7 +21,6 @@ import com.fasterxml.jackson.databind.JsonMappingException; import org.adrianwalker.multilinestring.Multiline; -import org.apache.metron.common.utils.JSONUtils; import org.junit.Test; import java.io.IOException; @@ -51,11 +50,28 @@ public class ProfileConfigTest { * The 'onlyif' field should default to 'true' when it is not specified. */ @Test - public void testOnlyIfDefault() throws IOException { - ProfileConfig profile = JSONUtils.INSTANCE.load(onlyIfDefault, ProfileConfig.class); + public void testFromJSONWithOnlyIfDefault() throws IOException { + ProfileConfig profile = ProfileConfig.fromJSON(onlyIfDefault); assertEquals("true", profile.getOnlyif()); } + /** + * Tests serializing the Profiler configuration to JSON. + */ + @Test + public void testToJSONWithOnlyIfDefault() throws Exception { + + // setup a profiler config to serialize + ProfileConfig expected = ProfileConfig.fromJSON(onlyIfDefault); + + // execute the test - serialize the config + String asJson = expected.toJSON(); + + // validate - deserialize to validate + ProfileConfig actual = ProfileConfig.fromJSON(asJson); + assertEquals(expected, actual); + } + /** * { * "foreach": "ip_src_addr", @@ -70,8 +86,8 @@ public void testOnlyIfDefault() throws IOException { * The 'name' of the profile must be defined. */ @Test(expected = JsonMappingException.class) - public void testNameMissing() throws IOException { - JSONUtils.INSTANCE.load(nameMissing, ProfileConfig.class); + public void testFromJSONWithNameMissing() throws IOException { + ProfileConfig.fromJSON(nameMissing); } /** @@ -88,8 +104,8 @@ public void testNameMissing() throws IOException { * The 'foreach' field must be defined. */ @Test(expected = JsonMappingException.class) - public void testForeachMissing() throws IOException { - JSONUtils.INSTANCE.load(foreachMissing, ProfileConfig.class); + public void testFromJSONWithForeachMissing() throws IOException { + ProfileConfig.fromJSON(foreachMissing); } /** @@ -106,8 +122,8 @@ public void testForeachMissing() throws IOException { * The 'result' field must be defined. */ @Test(expected = JsonMappingException.class) - public void testResultMissing() throws IOException { - JSONUtils.INSTANCE.load(resultMissing, ProfileConfig.class); + public void testFromJSONWithResultMissing() throws IOException { + ProfileConfig.fromJSON(resultMissing); } /** @@ -125,8 +141,8 @@ public void testResultMissing() throws IOException { * The 'result' field must contain the 'profile' expression used to store the profile measurement. */ @Test(expected = JsonMappingException.class) - public void testResultMissingProfileExpression() throws IOException { - JSONUtils.INSTANCE.load(resultMissingProfileExpression, ProfileConfig.class); + public void testFromJSONWithResultMissingProfileExpression() throws IOException { + ProfileConfig.fromJSON(resultMissingProfileExpression); } /** @@ -145,14 +161,31 @@ public void testResultMissingProfileExpression() throws IOException { * the 'profile' expression used to store the profile measurement. */ @Test - public void testResultWithExpression() throws IOException { - ProfileConfig profile = JSONUtils.INSTANCE.load(resultWithExpression, ProfileConfig.class); + public void testFromJSONWithResultWithExpression() throws IOException { + ProfileConfig profile = ProfileConfig.fromJSON(resultWithExpression); assertEquals("2 + 2", profile.getResult().getProfileExpressions().getExpression()); // no triage expressions expected assertEquals(0, profile.getResult().getTriageExpressions().getExpressions().size()); } + /** + * Tests serializing the Profiler configuration to JSON. + */ + @Test + public void testToJSONWithResultWithExpression() throws Exception { + + // setup a profiler config to serialize + ProfileConfig expected = ProfileConfig.fromJSON(resultWithExpression); + + // execute the test - serialize the config + String asJson = expected.toJSON(); + + // validate - deserialize to validate + ProfileConfig actual = ProfileConfig.fromJSON(asJson); + assertEquals(expected, actual); + } + /** * { * "profile": "test", @@ -170,14 +203,31 @@ public void testResultWithExpression() throws IOException { * The result's 'triage' field is optional. */ @Test - public void testResultWithProfileOnly() throws IOException { - ProfileConfig profile = JSONUtils.INSTANCE.load(resultWithProfileOnly, ProfileConfig.class); + public void testFromJSONWithResultWithProfileOnly() throws IOException { + ProfileConfig profile = ProfileConfig.fromJSON(resultWithProfileOnly); assertEquals("2 + 2", profile.getResult().getProfileExpressions().getExpression()); // no triage expressions expected assertEquals(0, profile.getResult().getTriageExpressions().getExpressions().size()); } + /** + * Tests serializing the Profiler configuration to JSON. + */ + @Test + public void testToJSONWithProfileOnly() throws Exception { + + // setup a profiler config to serialize + ProfileConfig expected = ProfileConfig.fromJSON(resultWithProfileOnly); + + // execute the test - serialize the config + String asJson = expected.toJSON(); + + // validate - deserialize to validate + ProfileConfig actual = ProfileConfig.fromJSON(asJson); + assertEquals(expected, actual); + } + /** * { * "profile": "test", @@ -199,10 +249,28 @@ public void testResultWithProfileOnly() throws IOException { * The result's 'triage' field can contain many named expressions. */ @Test - public void testResultWithTriage() throws IOException { - ProfileConfig profile = JSONUtils.INSTANCE.load(resultWithTriage, ProfileConfig.class); + public void testFromJSONWithResultWithTriage() throws IOException { + ProfileConfig profile = ProfileConfig.fromJSON(resultWithTriage); assertEquals("4 + 4", profile.getResult().getTriageExpressions().getExpression("eight")); assertEquals("8 + 8", profile.getResult().getTriageExpressions().getExpression("sixteen")); } + + /** + * Tests serializing the Profiler configuration to JSON. + */ + @Test + public void testToJSONWithResultWithTriage() throws Exception { + + // setup a profiler config to serialize + ProfileConfig expected = ProfileConfig.fromJSON(resultWithTriage); + + // execute the test - serialize the config + String asJson = expected.toJSON(); + + // validate - deserialize to validate + ProfileConfig actual = ProfileConfig.fromJSON(asJson); + assertEquals(expected, actual); + } + } diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/profiler/ProfilerConfigTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/profiler/ProfilerConfigTest.java index 2e73cde724..1a1181187e 100644 --- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/profiler/ProfilerConfigTest.java +++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/profiler/ProfilerConfigTest.java @@ -20,7 +20,6 @@ package org.apache.metron.common.configuration.profiler; import org.adrianwalker.multilinestring.Multiline; -import org.apache.metron.common.utils.JSONUtils; import org.junit.Test; import java.io.IOException; @@ -34,6 +33,33 @@ */ public class ProfilerConfigTest { + /** + * { + * "profiles": [ + * { + * "profile": "profile1", + * "foreach": "ip_src_addr", + * "init": { "count": "0" }, + * "update": { "count": "count + 1" }, + * "result": "count" + * } + * ] + * } + */ + @Multiline + private String profile; + + /** + * Tests deserializing the Profiler configuration using the fromJSON(...) method. + */ + @Test + public void testFromJSON() throws IOException { + ProfilerConfig conf = ProfilerConfig.fromJSON(profile); + + assertFalse(conf.getTimestampField().isPresent()); + assertEquals(1, conf.getProfiles().size()); + } + /** * { * "profiles": [ @@ -54,8 +80,8 @@ public class ProfilerConfigTest { * If no 'timestampField' is defined, it should not be present by default. */ @Test - public void testNoTimestampField() throws IOException { - ProfilerConfig conf = JSONUtils.INSTANCE.load(noTimestampField, ProfilerConfig.class); + public void testFromJSONWithNoTimestampField() throws IOException { + ProfilerConfig conf = ProfilerConfig.fromJSON(noTimestampField); assertFalse(conf.getTimestampField().isPresent()); } @@ -77,11 +103,12 @@ public void testNoTimestampField() throws IOException { private String timestampField; /** - * If no 'timestampField' is defined, it should not be present by default. + * Tests deserializing the Profiler configuration when the timestamp field is defined. */ @Test - public void testTimestampField() throws IOException { - ProfilerConfig conf = JSONUtils.INSTANCE.load(timestampField, ProfilerConfig.class); + public void testFromJSONWithTimestampField() throws IOException { + ProfilerConfig conf = ProfilerConfig.fromJSON(timestampField); + assertTrue(conf.getTimestampField().isPresent()); } @@ -108,13 +135,75 @@ public void testTimestampField() throws IOException { @Multiline private String twoProfiles; + @Test + public void testFromJSONTwoProfiles() throws IOException { + ProfilerConfig conf = ProfilerConfig.fromJSON(twoProfiles); + + assertEquals(2, conf.getProfiles().size()); + assertFalse(conf.getTimestampField().isPresent()); + } + /** - * The 'onlyif' field should default to 'true' when it is not specified. + * Tests serializing the Profiler configuration to JSON. */ @Test - public void testTwoProfiles() throws IOException { - ProfilerConfig conf = JSONUtils.INSTANCE.load(twoProfiles, ProfilerConfig.class); - assertEquals(2, conf.getProfiles().size()); + public void testToJSON() throws Exception { + + // setup a profiler config to serialize + ProfilerConfig expected = ProfilerConfig.fromJSON(profile); + + // execute the test - serialize the config + String asJson = expected.toJSON(); + + // validate - deserialize to validate + ProfilerConfig actual = ProfilerConfig.fromJSON(asJson); + assertEquals(expected, actual); } + /** + * { + * "profiles": [ + * { + * "profile": "profile1", + * "foreach": "ip_src_addr", + * "init": { "count": "0" }, + * "update": { "count": "count + 1" }, + * "result": { + * "profile": "count", + * "triage" : { "count": "count" } + * } + * } + * ] + * } + */ + @Multiline + private String profileWithTriageExpression; + + @Test + public void testToJSONWithTriageExpression() throws Exception { + + // setup a profiler config to serialize + ProfilerConfig expected = ProfilerConfig.fromJSON(profileWithTriageExpression); + + // execute the test - serialize the config + String asJson = expected.toJSON(); + + // validate - deserialize to validate + ProfilerConfig actual = ProfilerConfig.fromJSON(asJson); + assertEquals(expected, actual); + } + + @Test + public void testToJSONWithTwoProfiles() throws Exception { + + // setup a profiler config to serialize + ProfilerConfig expected = ProfilerConfig.fromJSON(twoProfiles); + + // execute the test - serialize the config + String asJson = expected.toJSON(); + + // validate - deserialize to validate + ProfilerConfig actual = ProfilerConfig.fromJSON(asJson); + assertEquals(expected, actual); + } } From bd59376c4ebee2c3db4634023030cb846e120d7a Mon Sep 17 00:00:00 2001 From: Nick Allen Date: Sat, 7 Apr 2018 11:47:28 -0400 Subject: [PATCH 2/2] Using commons-lang for equals,hashcode, and tostring to match ProfileConfig class --- .../profiler/ProfilerConfig.java | 34 +++++++++++++------ 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfilerConfig.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfilerConfig.java index 451c86bda9..e4fa99a85a 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfilerConfig.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfilerConfig.java @@ -20,6 +20,9 @@ import com.fasterxml.jackson.annotation.JsonGetter; import com.fasterxml.jackson.annotation.JsonSetter; import com.fasterxml.jackson.core.JsonProcessingException; +import org.apache.commons.lang.builder.EqualsBuilder; +import org.apache.commons.lang.builder.HashCodeBuilder; +import org.apache.commons.lang3.builder.ToStringBuilder; import org.apache.metron.common.utils.JSONUtils; import org.codehaus.jackson.map.annotate.JsonSerialize; import org.codehaus.jackson.map.annotate.JsonSerialize.Inclusion; @@ -93,26 +96,35 @@ public ProfilerConfig withTimestampField(Optional timestampField) { @Override public String toString() { - return "ProfilerConfig{" + - "profiles=" + profiles + - ", timestampField='" + timestampField + '\'' + - '}'; + return new ToStringBuilder(this) + .append("profiles", profiles) + .append("timestampField", timestampField) + .toString(); } @Override public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; + if (this == o) { + return true; + } + + if (o == null || getClass() != o.getClass()) { + return false; + } + ProfilerConfig that = (ProfilerConfig) o; - if (profiles != null ? !profiles.equals(that.profiles) : that.profiles != null) return false; - return timestampField != null ? timestampField.equals(that.timestampField) : that.timestampField == null; + return new EqualsBuilder() + .append(profiles, that.profiles) + .append(timestampField, that.timestampField) + .isEquals(); } @Override public int hashCode() { - int result = profiles != null ? profiles.hashCode() : 0; - result = 31 * result + (timestampField != null ? timestampField.hashCode() : 0); - return result; + return new HashCodeBuilder(17, 37) + .append(profiles) + .append(timestampField) + .toHashCode(); } /**