Improved datetime unit transformer shortcuts - Part 2 (#319)

core/src/main/scala/com/salesforce/op/dsl/RichListFeature.scala

@@ -199,6 +199,14 @@ trait RichListFeature {
    */
   implicit class RichDateListFeature(val f: FeatureLike[DateList]) {
 
+    /**
+     * Convert to specified time period
+     * @param period type of [[TimePeriod]] to convert date feature to
+     * @return OPVector feature of time period values
+     */
+    def toTimePeriod(period: TimePeriod): FeatureLike[OPVector] =
+      new TimePeriodListTransformer[DateList](period).setInput(f).getOutput()
+
     /**
      * Apply DateList vectorizer: Converts a sequence of DateLists features into a vector feature.
      *
@@ -240,6 +248,14 @@ trait RichListFeature {
    */
   implicit class RichDateTimeListFeature(val f: FeatureLike[DateTimeList]) {
 
+    /**
+     * Convert to specified time period
+     * @param period type of [[TimePeriod]] to convert date feature to
+     * @return OPVector feature of time period values
+     */
+    def toTimePeriod(period: TimePeriod): FeatureLike[OPVector] =
+      new TimePeriodListTransformer[DateTimeList](period).setInput(f).getOutput()
+
     /**
      * Apply DateList vectorizer: Converts a sequence of DateTimeLists features into a vector feature.
      *

core/src/main/scala/com/salesforce/op/dsl/RichMapFeature.scala

@@ -702,6 +702,14 @@ trait RichMapFeature {
    */
   implicit class RichDateMapFeature(val f: FeatureLike[DateMap]) {
 
+    /**
+     * Convert to specified time period
+     * @param period type of [[TimePeriod]] to convert date feature to
+     * @return Integer map feature of time period values
+     */
+    def toTimePeriod(period: TimePeriod): FeatureLike[IntegralMap] =
+      new TimePeriodMapTransformer[DateMap](period).setInput(f).getOutput()
+
     /**
      * transforms a DateMap field into a series of cartesian coordinate representation
      * of an extracted time period on the unit circle
@@ -781,6 +789,13 @@ trait RichMapFeature {
    */
   implicit class RichDateTimeMapFeature(val f: FeatureLike[DateTimeMap]) {
 
+    /**
+     * Convert to specified time period
+     * @param period type of [[TimePeriod]] to convert date feature to
+     * @return Integer map feature of time period values
+     */
+    def toTimePeriod(period: TimePeriod): FeatureLike[IntegralMap] =
+      new TimePeriodMapTransformer[DateTimeMap](period).setInput(f).getOutput()
 
     /**
      * transforms a DateTimeMap field into a series of cartesian coordinate representation

core/src/main/scala/com/salesforce/op/stages/impl/feature/TimePeriodListTransformer.scala

@@ -0,0 +1,57 @@
+/*
+ * Copyright (c) 2017, Salesforce.com, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice, this
+ *   list of conditions and the following disclaimer.
+ *
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ *
+ * * Neither the name of the copyright holder nor the names of its
+ *   contributors may be used to endorse or promote products derived from
+ *   this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package com.salesforce.op.stages.impl.feature
+
+import com.salesforce.op.UID
+import com.salesforce.op.features.types._
+import com.salesforce.op.stages.base.unary.UnaryTransformer
+
+import scala.reflect.runtime.universe.TypeTag
+
+/**
+ * TimePeriodMapTransformer extracts one of a set of time periods from a date/datetime list
+ *
+ * @param period        time period to extract from date
+ * @param uid           uid for instance
+ * @param tti           type tag for input
+ * @tparam I            input feature type
+ */
+class TimePeriodListTransformer[I <: DateList]
+(
+  val period: TimePeriod,
+  uid: String = UID[TimePeriodListTransformer[_]]
+)(
+  implicit override val tti: TypeTag[I]
+) extends UnaryTransformer[I, OPVector](operationName = "dateListToTimePeriod", uid = uid) {
+
+  override def transformFn: I => OPVector =
+    (i: I) => i.value.map(t => period.extractFromTime(t).toDouble).toOPVector
+}

core/src/main/scala/com/salesforce/op/stages/impl/feature/TimePeriodMapTransformer.scala

@@ -0,0 +1,57 @@
+/*
+ * Copyright (c) 2017, Salesforce.com, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice, this
+ *   list of conditions and the following disclaimer.
+ *
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ *
+ * * Neither the name of the copyright holder nor the names of its
+ *   contributors may be used to endorse or promote products derived from
+ *   this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package com.salesforce.op.stages.impl.feature
+
+import com.salesforce.op.UID
+import com.salesforce.op.features.types._
+import com.salesforce.op.stages.base.unary.UnaryTransformer
+
+import scala.reflect.runtime.universe.TypeTag
+
+/**
+ * TimePeriodMapTransformer extracts one of a set of time periods from a date/datetime map
+ *
+ * @param period        time period to extract from date
+ * @param uid           uid for instance
+ * @param tti           type tag for input
+ * @tparam I            input feature type
+ */
+class TimePeriodMapTransformer[I <: DateMap]
+(
+  val period: TimePeriod,
+  uid: String = UID[TimePeriodMapTransformer[_]]
+)(
+  implicit override val tti: TypeTag[I]
+) extends UnaryTransformer[I, IntegralMap](operationName = "dateMapToTimePeriod", uid = uid) {
+
+  override def transformFn: I => IntegralMap =
+    (i: I) => i.value.mapValues(t => period.extractFromTime(t).toLong).toIntegralMap
+}

core/src/test/scala/com/salesforce/op/stages/impl/feature/TimePeriodListTransformerTest.scala

@@ -0,0 +1,75 @@
+/*
+ * Copyright (c) 2017, Salesforce.com, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice, this
+ *   list of conditions and the following disclaimer.
+ *
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ *
+ * * Neither the name of the copyright holder nor the names of its
+ *   contributors may be used to endorse or promote products derived from
+ *   this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package com.salesforce.op.stages.impl.feature
+
+import com.salesforce.op.features.FeatureLike
+import com.salesforce.op.features.types._
+import com.salesforce.op.test.{OpTransformerSpec, TestFeatureBuilder}
+import com.salesforce.op.utils.date.DateTimeUtils
+import com.salesforce.op.utils.spark.RichDataset._
+import org.apache.spark.ml.Transformer
+import org.joda.time.{DateTime => JDateTime}
+import org.junit.runner.RunWith
+import org.scalatest.junit.JUnitRunner
+
+@RunWith(classOf[JUnitRunner])
+class TimePeriodListTransformerTest extends OpTransformerSpec[OPVector, TimePeriodListTransformer[DateList]] {
+
+  val dateList: DateList = Seq[Long](
+    new JDateTime(1879, 3, 14, 0, 0, DateTimeUtils.DefaultTimeZone).getMillis,
+    new JDateTime(1955, 11, 12, 10, 4, DateTimeUtils.DefaultTimeZone).getMillis,
+    new JDateTime(1999, 3, 8, 12, 0, DateTimeUtils.DefaultTimeZone).getMillis,
+    new JDateTime(2019, 4, 30, 13, 0, DateTimeUtils.DefaultTimeZone).getMillis
+  ).toDateList
+
+  val (inputData, f1) = TestFeatureBuilder(Seq(dateList))
+
+  override val transformer: TimePeriodListTransformer[DateList] =
+    new TimePeriodListTransformer(TimePeriod.DayOfMonth).setInput(f1)
+
+  override val expectedResult: Seq[OPVector] = Seq(Seq(14, 12, 8, 30).map(_.toDouble).toVector.toOPVector)
+
+  it should "transform with rich shortcuts" in {
+    val dlist = List(new JDateTime(1879, 3, 14, 0, 0, DateTimeUtils.DefaultTimeZone).getMillis)
+    val (inputData2, d1, d2) = TestFeatureBuilder(
+      Seq[(DateList, DateTimeList)]((dlist.toDateList, dlist.toDateTimeList))
+    )
+
+    def assertFeature(feature: FeatureLike[OPVector], expected: Seq[OPVector]): Unit = {
+      val transformed = feature.originStage.asInstanceOf[Transformer].transform(inputData2)
+      val actual = transformed.collect(feature)
+      actual shouldBe expected
+    }
+
+    assertFeature(d1.toTimePeriod(TimePeriod.DayOfMonth), Seq(Vector(14.0).toOPVector))
+    assertFeature(d2.toTimePeriod(TimePeriod.DayOfMonth), Seq(Vector(14.0).toOPVector))
+  }
+}

core/src/test/scala/com/salesforce/op/stages/impl/feature/TimePeriodMapTransformerTest.scala

@@ -0,0 +1,82 @@
+/*
+ * Copyright (c) 2017, Salesforce.com, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice, this
+ *   list of conditions and the following disclaimer.
+ *
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ *
+ * * Neither the name of the copyright holder nor the names of its
+ *   contributors may be used to endorse or promote products derived from
+ *   this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package com.salesforce.op.stages.impl.feature
+
+import com.salesforce.op.features.FeatureLike
+import com.salesforce.op.features.types._
+import com.salesforce.op.test.{OpTransformerSpec, TestFeatureBuilder}
+import com.salesforce.op.utils.date.DateTimeUtils
+import com.salesforce.op.utils.spark.RichDataset._
+import org.apache.spark.ml.Transformer
+import org.joda.time.{DateTime => JDateTime}
+import org.junit.runner.RunWith
+import org.scalatest.junit.JUnitRunner
+
+@RunWith(classOf[JUnitRunner])
+class TimePeriodMapTransformerTest extends OpTransformerSpec[IntegralMap, TimePeriodMapTransformer[DateMap]] {
+
+  val names: Seq[String] = Seq("n1", "n2", "n3", "n4")
+
+  val dates: Seq[Long] = Seq(
+    new JDateTime(1879, 3, 14, 0, 0, DateTimeUtils.DefaultTimeZone).getMillis,
+    new JDateTime(1955, 11, 12, 10, 4, DateTimeUtils.DefaultTimeZone).getMillis,
+    new JDateTime(1999, 3, 8, 12, 0, DateTimeUtils.DefaultTimeZone).getMillis,
+    new JDateTime(2019, 4, 30, 13, 0, DateTimeUtils.DefaultTimeZone).getMillis
+  )
+
+  val dateMap: DateMap = names.zip(dates).toMap.toDateMap
+
+  val (inputData, f1) = TestFeatureBuilder(Seq[DateMap](dateMap))
+
+  override val transformer: TimePeriodMapTransformer[DateMap] =
+    new TimePeriodMapTransformer(TimePeriod.DayOfMonth).setInput(f1)
+
+  override val expectedResult: Seq[IntegralMap] = Seq(
+    names.zip(Seq(14L, 12L, 8L, 30L)).toMap.toIntegralMap
+  )
+
+  it should "transform with rich shortcuts" in {
+    val n = "n1"
+    val dmap = Map(n -> new JDateTime(1879, 3, 14, 0, 0, DateTimeUtils.DefaultTimeZone).getMillis)
+    val (inputData2, d1, d2) = TestFeatureBuilder(
+      Seq[(DateMap, DateTimeMap)]((dmap.toDateMap, dmap.toDateTimeMap))
+    )
+
+    def assertFeature(feature: FeatureLike[IntegralMap], expected: Seq[IntegralMap]): Unit = {
+      val transformed = feature.originStage.asInstanceOf[Transformer].transform(inputData2)
+      val actual = transformed.collect(feature)
+      actual shouldBe expected
+    }
+
+    assertFeature(d1.toTimePeriod(TimePeriod.DayOfMonth), Seq(IntegralMap(Map(n -> 14))))
+    assertFeature(d2.toTimePeriod(TimePeriod.DayOfMonth), Seq(IntegralMap(Map(n -> 14))))
+  }
+}