Support sql expression in FDSExtract (#1112)

feathr-ai · Mar 9, 2023 · cb29dd9 · cb29dd9
1 parent e42bb19
commit cb29dd9
Show file tree

Hide file tree

Showing 3 changed files with 92 additions and 14 deletions.
diff --git a/feathr-impl/src/main/scala/com/linkedin/feathr/offline/job/FeatureTransformation.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/job/FeatureTransformation.scala
@@ -1359,7 +1359,7 @@ private[offline] object FeatureTransformation {
   def parseMultiDimTensorExpr(featureDef: String): String = {
     // String char should be one more than the len of the keyword to account for '('. The end should be 1 less than length of feature string
     // to account for ')'.
-    featureDef.substring(featureDef.indexOf("(") + 1, featureDef.indexOf(")"))
+    featureDef.substring(featureDef.indexOf("(") + 1, featureDef.lastIndexOf(")"))
   }
 
 

diff --git a/feathr-impl/src/test/resources/anchorAndDerivations/nullValueSource.avro.json b/feathr-impl/src/test/resources/anchorAndDerivations/nullValueSource.avro.json
@@ -9,16 +9,81 @@
       "name": "value",
       "type": ["null", "int"],
       "default": null
-    }]
+    },
+      {
+        "name" : "denseValue",
+        "type" : [ {
+          "type" : "array",
+          "items" : [ {
+            "type" : "array",
+            "items" : [ "string", "null" ]
+          }, "null" ]
+        }, "null" ]
+      } ]
   },
   "data": [{
     "mId": 1,
-    "value": { "int": 1 }
+    "value": { "int": 1 },
+    "denseValue" : {
+      "array" : [ {
+        "array" : [ {
+          "string" : "aa"
+        }, {
+          "string" : "bb"
+        }, {
+          "string" : "cc"
+        }, {
+          "string" : "dd"
+        }, {
+          "string" : "ee"
+        } ]
+      }, {
+        "array" : [ {
+          "string" : "a"
+        }, {
+          "string" : "a"
+        }, {
+          "string" : "a"
+        }, {
+          "string" : "a"
+        }, {
+          "string" : "a"
+        } ]
+      } ]
+    }
   }, {
     "mId": 2,
-    "value": null
+    "value": null,
+    "denseValue" : null
   }, {
     "mId": 3,
-    "value": { "int": 3 }
-  }]
-}
+    "value": { "int": 3 },
+    "denseValue" : {
+      "array" : [ {
+          "array" : [ {
+            "string" : "aa"
+          }, {
+            "string" : "bb"
+          }, {
+            "string" : "cc"
+          }, {
+            "string" : "dd"
+          }, {
+            "string" : "ee"
+          } ]
+        }, {
+          "array" : [ {
+          "string" : "a"
+        }, {
+          "string" : "a"
+        }, {
+          "string" : "a"
+        }, {
+          "string" : "a"
+        }, {
+          "string" : "a"
+        } ]
+     }]
+    }
+    }]
+  }
diff --git a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/AnchoredFeaturesIntegTest.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/AnchoredFeaturesIntegTest.scala
@@ -406,8 +406,17 @@ class AnchoredFeaturesIntegTest extends FeathrIntegTest {
           |    source: "anchorAndDerivations/nullValueSource.avro.json"
           |    key.sqlExpr: mId
           |    features: {
-          |      featureWithNull.def.sqlExpr: FDSExtract(value)
-          |    }
+          |      featureWithNull {
+            |      def.sqlExpr: "FDSExtract(coalesce(denseValue, ARRAY(ARRAY(\"aa\", \"bb\", \"cc\", \"dd\", \"ee\"), ARRAY(\"UNK\", \"UNK\", \"UNK\", \"UNK\", \"UNK\")) ))"
+            |      type:{
+            |          type: TENSOR
+            |           tensorCategory: DENSE
+            |           shape: [2,5]
+            |           dimensionType: [INT, INT]
+            |           valType: STRING
+            |          }
+          |         }
+          |       }
           |  }
           |}
         """.stripMargin,
@@ -423,21 +432,25 @@ class AnchoredFeaturesIntegTest extends FeathrIntegTest {
             // a_id
             "1",
             // featureWithNull
-            1),
+            mutable.WrappedArray.make(Array(Array("aa", "bb", "cc", "dd", "ee"), Array("a", "a", "a", "a", "a"))),
+          ),
           Row(
             // a_id
             "2",
-            // featureWithNull
-            null),
+            // f3eatureWithNull
+            mutable.WrappedArray.make(Array(Array("aa", "bb", "cc", "dd", "ee"), Array("UNK", "UNK", "UNK", "UNK", "UNK")))
+          ),
           Row(
             // a_id
             "3",
             // featureWithNull
-            3))),
+            mutable.WrappedArray.make(Array(Array("aa", "bb", "cc", "dd", "ee"), Array("a", "a", "a", "a", "a")),
+              )))),
       StructType(
         List(
           StructField("a_id", StringType, true),
-          StructField("featureWithNull", IntegerType, true))))
+          StructField("featureWithNull", ArrayType(ArrayType(StringType, true), true), true)
+        )))
 
     def cmpFunc(row: Row): String = row.get(0).toString