Merge pull request #296 from djfreels/develop

StepParameter level json4s Formats
Acxiom · Feb 7, 2022 · d509269 · d509269
2 parents ac80ff5 + 23bf176
commit d509269
Show file tree

Hide file tree

Showing 13 changed files with 21 additions and 18 deletions.
diff --git a/metalus-application/pom.xml b/metalus-application/pom.xml
@@ -9,7 +9,7 @@
     <parent>
         <groupId>com.acxiom</groupId>
         <artifactId>metalus</artifactId>
-        <version>1.8.5-SNAPSHOT</version>
+        <version>1.8.6-SNAPSHOT</version>
     </parent>
     <dependencies>
         <dependency>

diff --git a/metalus-aws/pom.xml b/metalus-aws/pom.xml
@@ -9,7 +9,7 @@
     <parent>
         <groupId>com.acxiom</groupId>
         <artifactId>metalus</artifactId>
-        <version>1.8.5-SNAPSHOT</version>
+        <version>1.8.6-SNAPSHOT</version>
     </parent>
 
     <properties>

diff --git a/metalus-common/pom.xml b/metalus-common/pom.xml
@@ -9,7 +9,7 @@
     <parent>
         <groupId>com.acxiom</groupId>
         <artifactId>metalus</artifactId>
-        <version>1.8.5-SNAPSHOT</version>
+        <version>1.8.6-SNAPSHOT</version>
     </parent>
 
     <dependencies>

diff --git a/metalus-core/pom.xml b/metalus-core/pom.xml
@@ -9,7 +9,7 @@
     <parent>
         <groupId>com.acxiom</groupId>
         <artifactId>metalus</artifactId>
-        <version>1.8.5-SNAPSHOT</version>
+        <version>1.8.6-SNAPSHOT</version>
     </parent>
 
     <properties>

diff --git a/metalus-core/src/main/scala/com/acxiom/pipeline/PipelineStep.scala b/metalus-core/src/main/scala/com/acxiom/pipeline/PipelineStep.scala
@@ -1,9 +1,10 @@
 package com.acxiom.pipeline
 
 import com.acxiom.pipeline.PipelineStepMessageType.PipelineStepMessageType
-
 import java.util.Date
 
+import com.acxiom.pipeline.applications.Json4sSerializers
+
 /**
   * Metadata about the next step in the pipeline process.
   *
@@ -51,7 +52,8 @@ case class Parameter(`type`: Option[String] = None,
                      value: Option[Any] = None,
                      className: Option[String] = None,
                      parameterType: Option[String] = None,
-                     description: String = "")
+                     description: String = "",
+                     json4sSerializers: Option[Json4sSerializers] = None)
 
 /**
   * This class contains the execution information for a Step

diff --git a/metalus-core/src/main/scala/com/acxiom/pipeline/PipelineStepMapper.scala b/metalus-core/src/main/scala/com/acxiom/pipeline/PipelineStepMapper.scala
@@ -1,5 +1,6 @@
 package com.acxiom.pipeline
 
+import com.acxiom.pipeline.applications.ApplicationUtils
 import com.acxiom.pipeline.utils.{DriverUtils, ReflectionUtils, ScalaScriptEngine}
 import org.apache.log4j.Logger
 import org.json4s.Formats
@@ -267,8 +268,9 @@ trait PipelineStepMapper {
                                  parameter: Parameter,
                                  pipelineContext: PipelineContext): Option[Any] = {
     val workingMap = map.asInstanceOf[Map[String, Any]]
+    implicit val formats: Formats = parameter.json4sSerializers.map(js => ApplicationUtils.getJson4sFormats(Some(js)))
+      .getOrElse(pipelineContext.getJson4sFormats)
     Some(if (parameter.className.isDefined && parameter.className.get.nonEmpty) {
-      implicit val formats: Formats = pipelineContext.getJson4sFormats
       // Skip the embedded variable mapping if this is a step-group pipeline parameter
       if (workingMap.getOrElse("category", "pipeline").asInstanceOf[String] == "step-group") {
         DriverUtils.parseJson(Serialization.write(workingMap), parameter.className.get)
@@ -291,8 +293,9 @@ trait PipelineStepMapper {
     */
   private def handleListParameter(list: List[_], parameter: Parameter, pipelineContext: PipelineContext): Option[Any] = {
     val dropNone = pipelineContext.getGlobalAs[Boolean]("dropNoneFromLists").getOrElse(true)
+    implicit val formats: Formats = parameter.json4sSerializers.map(js => ApplicationUtils.getJson4sFormats(Some(js)))
+      .getOrElse(pipelineContext.getJson4sFormats)
     Some(if (parameter.className.isDefined && parameter.className.get.nonEmpty) {
-      implicit val formats: Formats = pipelineContext.getJson4sFormats
       list.map(value =>
         DriverUtils.parseJson(Serialization.write(mapEmbeddedVariables(value.asInstanceOf[Map[String, Any]], pipelineContext)), parameter.className.get))
     } else if (list.nonEmpty && list.head.isInstanceOf[Map[_, _]]) {
@@ -325,8 +328,7 @@ trait PipelineStepMapper {
     * @param pipelineContext The pipelineContext
     * @return A map with substituted values
     */
-  private[pipeline] def mapEmbeddedVariables(classMap: Map[String, Any], pipelineContext: PipelineContext): Map[String, Any] = {
-    implicit val formats: Formats = pipelineContext.getJson4sFormats
+  private[pipeline] def mapEmbeddedVariables(classMap: Map[String, Any], pipelineContext: PipelineContext)(implicit formats: Formats): Map[String, Any] = {
     classMap.foldLeft(classMap)((map, entry) => {
       entry._2 match {
         case s: String if containsSpecialCharacters(s) =>
@@ -592,7 +594,6 @@ trait PipelineStepMapper {
   }
 
   private def mapByValue(value: Option[String], parameter: Parameter, pipelineContext: PipelineContext): Any = {
-    implicit val formats: Formats = pipelineContext.getJson4sFormats
     if (value.isDefined) {
       value.get
     } else if (parameter.defaultValue.isDefined) {

diff --git a/metalus-delta/pom.xml b/metalus-delta/pom.xml
@@ -12,7 +12,7 @@
     <parent>
         <groupId>com.acxiom</groupId>
         <artifactId>metalus</artifactId>
-        <version>1.8.5-SNAPSHOT</version>
+        <version>1.8.6-SNAPSHOT</version>
     </parent>
 
     <dependencies>

diff --git a/metalus-examples/pom.xml b/metalus-examples/pom.xml
@@ -9,7 +9,7 @@
     <parent>
         <groupId>com.acxiom</groupId>
         <artifactId>metalus</artifactId>
-        <version>1.8.5-SNAPSHOT</version>
+        <version>1.8.6-SNAPSHOT</version>
     </parent>
 
     <properties>

diff --git a/metalus-gcp/pom.xml b/metalus-gcp/pom.xml
@@ -9,7 +9,7 @@
     <parent>
         <groupId>com.acxiom</groupId>
         <artifactId>metalus</artifactId>
-        <version>1.8.5-SNAPSHOT</version>
+        <version>1.8.6-SNAPSHOT</version>
     </parent>
 
     <dependencyManagement>

diff --git a/metalus-kafka/pom.xml b/metalus-kafka/pom.xml
@@ -9,7 +9,7 @@
     <parent>
         <groupId>com.acxiom</groupId>
         <artifactId>metalus</artifactId>
-        <version>1.8.5-SNAPSHOT</version>
+        <version>1.8.6-SNAPSHOT</version>
     </parent>
 
     <properties>

diff --git a/metalus-mongo/pom.xml b/metalus-mongo/pom.xml
@@ -9,7 +9,7 @@
     <parent>
         <groupId>com.acxiom</groupId>
         <artifactId>metalus</artifactId>
-        <version>1.8.5-SNAPSHOT</version>
+        <version>1.8.6-SNAPSHOT</version>
     </parent>
 
     <dependencies>

diff --git a/metalus-utils/pom.xml b/metalus-utils/pom.xml
@@ -9,7 +9,7 @@
     <parent>
         <groupId>com.acxiom</groupId>
         <artifactId>metalus</artifactId>
-        <version>1.8.5-SNAPSHOT</version>
+        <version>1.8.6-SNAPSHOT</version>
     </parent>
 
     <properties>

diff --git a/pom.xml b/pom.xml
@@ -3,7 +3,7 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>com.acxiom</groupId>
     <artifactId>metalus</artifactId>
-    <version>1.8.5-SNAPSHOT</version>
+    <version>1.8.6-SNAPSHOT</version>
     <name>${project.artifactId}</name>
     <packaging>pom</packaging>
     <description>Metalus Pipeline Library</description>