fix: use native hashing algorithm and fix bug

feathr-ai · Jun 28, 2023 · ff9bd8d · ff9bd8d
1 parent 869c31a
commit ff9bd8d
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/feathr-impl/src/main/scala/com/linkedin/feathr/offline/swa/SlidingWindowFeatureUtils.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/swa/SlidingWindowFeatureUtils.scala
@@ -187,7 +187,7 @@ private[offline] object SlidingWindowFeatureUtils {
         val rewrittenDef = s"CASE WHEN ${featureDef} IS NOT NULL THEN 1 ELSE 0 END"
         new CountAggregate(rewrittenDef)
       case AggregationType.COUNT_DISTINCT =>
-        var rewrittenDef = s"CASE WHEN ${featureDef} IS NOT NULL THEN CAST(CONV(MD5(${featureDef}), 16, 10) AS INT)  ELSE 0 END"
+        val rewrittenDef = s"CASE WHEN ${featureDef} IS NOT NULL THEN hash(${featureDef}) ELSE 0 END"
         new CountDistinctAggregate(rewrittenDef)
       case AggregationType.AVG => new AvgAggregate(featureDef)
       case AggregationType.MAX => new MaxAggregate(featureDef)