From f3f1fd2dcd413c7fc79f5a42849701d54b44724b Mon Sep 17 00:00:00 2001 From: Chris Rupley Date: Mon, 15 Apr 2019 16:01:01 -0700 Subject: [PATCH 01/11] match up names for OpLDA and test --- .../scala/com/salesforce/op/stages/impl/feature/OpLdaTest.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/test/scala/com/salesforce/op/stages/impl/feature/OpLdaTest.scala b/core/src/test/scala/com/salesforce/op/stages/impl/feature/OpLdaTest.scala index 903b98aadf..a1ccd078de 100644 --- a/core/src/test/scala/com/salesforce/op/stages/impl/feature/OpLdaTest.scala +++ b/core/src/test/scala/com/salesforce/op/stages/impl/feature/OpLdaTest.scala @@ -42,7 +42,7 @@ import org.scalatest.{Assertions, FlatSpec, Matchers} @RunWith(classOf[JUnitRunner]) -class OpLdaTest extends FlatSpec with TestSparkContext { +class OpLDATest extends FlatSpec with TestSparkContext { val inputData = Seq( (0.0, Vectors.sparse(11, Array(0, 1, 2, 4, 5, 6, 7, 10), Array(1.0, 2.0, 6.0, 2.0, 3.0, 1.0, 1.0, 3.0))), From 1bf74c08e84a65998583722ddb87d45dff1b1338 Mon Sep 17 00:00:00 2001 From: Chris Rupley Date: Mon, 15 Apr 2019 16:02:12 -0700 Subject: [PATCH 02/11] rename OpLDATest everywhere --- .../op/stages/impl/feature/{OpLdaTest.scala => OpLDATest.scala} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename core/src/test/scala/com/salesforce/op/stages/impl/feature/{OpLdaTest.scala => OpLDATest.scala} (98%) diff --git a/core/src/test/scala/com/salesforce/op/stages/impl/feature/OpLdaTest.scala b/core/src/test/scala/com/salesforce/op/stages/impl/feature/OpLDATest.scala similarity index 98% rename from core/src/test/scala/com/salesforce/op/stages/impl/feature/OpLdaTest.scala rename to core/src/test/scala/com/salesforce/op/stages/impl/feature/OpLDATest.scala index a1ccd078de..df35c4192e 100644 --- a/core/src/test/scala/com/salesforce/op/stages/impl/feature/OpLdaTest.scala +++ b/core/src/test/scala/com/salesforce/op/stages/impl/feature/OpLDATest.scala @@ -78,7 +78,7 @@ class OpLDATest extends FlatSpec with TestSparkContext { .toSeq .map(_.getAs[Vector](0)) - Spec[OpLdaTest] should "convert document term vectors into topic vectors" in { + Spec[OpLDATest] should "convert document term vectors into topic vectors" in { val f2Vec = new OpLDA().setInput(f2).setK(k).setSeed(seed).setMaxIter(maxIter) val testTransformedData = f2Vec.fit(inputDS).transform(inputDS) val output = f2Vec.getOutput() From 9a5953a83977e8bd3a692d3a966d9f17691b08d5 Mon Sep 17 00:00:00 2001 From: Chris Rupley Date: Mon, 15 Apr 2019 16:05:09 -0700 Subject: [PATCH 03/11] rename StandardScalar test to match --- ...tandardScalerTest.scala => OpScalarStandardScalerTest.scala} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename core/src/test/scala/com/salesforce/op/stages/impl/feature/{OpStandardScalerTest.scala => OpScalarStandardScalerTest.scala} (99%) diff --git a/core/src/test/scala/com/salesforce/op/stages/impl/feature/OpStandardScalerTest.scala b/core/src/test/scala/com/salesforce/op/stages/impl/feature/OpScalarStandardScalerTest.scala similarity index 99% rename from core/src/test/scala/com/salesforce/op/stages/impl/feature/OpStandardScalerTest.scala rename to core/src/test/scala/com/salesforce/op/stages/impl/feature/OpScalarStandardScalerTest.scala index 583556699c..a74ba04443 100644 --- a/core/src/test/scala/com/salesforce/op/stages/impl/feature/OpStandardScalerTest.scala +++ b/core/src/test/scala/com/salesforce/op/stages/impl/feature/OpScalarStandardScalerTest.scala @@ -45,7 +45,7 @@ import org.scalatest.junit.JUnitRunner @RunWith(classOf[JUnitRunner]) -class OpStandardScalerTest extends FlatSpec with TestSparkContext { +class OpScalarStandardScalerTest extends FlatSpec with TestSparkContext { import spark.implicits._ // TODO: use TestFeatureBuilder instead From 26058eebb04dae9b7021c7a6028d8e0cb8836140 Mon Sep 17 00:00:00 2001 From: Chris Rupley Date: Mon, 15 Apr 2019 16:07:14 -0700 Subject: [PATCH 04/11] rename TextMapPivotVectorizer test to match --- ...pVectorizerTest.scala => TextMapPivotVectorizerTest.scala} | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename core/src/test/scala/com/salesforce/op/stages/impl/feature/{TextMapVectorizerTest.scala => TextMapPivotVectorizerTest.scala} (99%) diff --git a/core/src/test/scala/com/salesforce/op/stages/impl/feature/TextMapVectorizerTest.scala b/core/src/test/scala/com/salesforce/op/stages/impl/feature/TextMapPivotVectorizerTest.scala similarity index 99% rename from core/src/test/scala/com/salesforce/op/stages/impl/feature/TextMapVectorizerTest.scala rename to core/src/test/scala/com/salesforce/op/stages/impl/feature/TextMapPivotVectorizerTest.scala index fb388f9e9c..42f8c1203a 100644 --- a/core/src/test/scala/com/salesforce/op/stages/impl/feature/TextMapVectorizerTest.scala +++ b/core/src/test/scala/com/salesforce/op/stages/impl/feature/TextMapPivotVectorizerTest.scala @@ -44,9 +44,9 @@ import org.slf4j.LoggerFactory @RunWith(classOf[JUnitRunner]) -class TextMapVectorizerTest extends FlatSpec with TestSparkContext with AttributeAsserts { +class TextMapPivotVectorizerTest extends FlatSpec with TestSparkContext with AttributeAsserts { - val log = LoggerFactory.getLogger(classOf[TextMapVectorizerTest]) + val log = LoggerFactory.getLogger(classOf[TextMapPivotVectorizerTest]) lazy val (dataSet, top, bot) = TestFeatureBuilder("top", "bot", Seq( From ddc8de1441bcf1bd716223f13ab8f3f79fb30076 Mon Sep 17 00:00:00 2001 From: Chris Rupley Date: Mon, 15 Apr 2019 16:09:48 -0700 Subject: [PATCH 05/11] rename Transmogrifier test to match --- .../{TransmogrifyTest.scala => TransmogrifierTest.scala} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename core/src/test/scala/com/salesforce/op/stages/impl/feature/{TransmogrifyTest.scala => TransmogrifierTest.scala} (97%) diff --git a/core/src/test/scala/com/salesforce/op/stages/impl/feature/TransmogrifyTest.scala b/core/src/test/scala/com/salesforce/op/stages/impl/feature/TransmogrifierTest.scala similarity index 97% rename from core/src/test/scala/com/salesforce/op/stages/impl/feature/TransmogrifyTest.scala rename to core/src/test/scala/com/salesforce/op/stages/impl/feature/TransmogrifierTest.scala index dd00952311..54df05a7b5 100644 --- a/core/src/test/scala/com/salesforce/op/stages/impl/feature/TransmogrifyTest.scala +++ b/core/src/test/scala/com/salesforce/op/stages/impl/feature/TransmogrifierTest.scala @@ -42,7 +42,7 @@ import org.scalatest.FlatSpec import org.scalatest.junit.JUnitRunner @RunWith(classOf[JUnitRunner]) -class TransmogrifyTest extends FlatSpec with PassengerSparkFixtureTest with AttributeAsserts { +class TransmogrifierTest extends FlatSpec with PassengerSparkFixtureTest with AttributeAsserts { val inputFeatures = Array[OPFeature](heightNoWindow, weight, gender) From 73c978afdf8f89e739e1fd3e1af2faedcc37cfeb Mon Sep 17 00:00:00 2001 From: Chris Rupley Date: Mon, 15 Apr 2019 17:07:07 -0700 Subject: [PATCH 06/11] add test --- .../impl/feature/RealNNVectorizerTest.scala | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 core/src/test/scala/com/salesforce/op/stages/impl/feature/RealNNVectorizerTest.scala diff --git a/core/src/test/scala/com/salesforce/op/stages/impl/feature/RealNNVectorizerTest.scala b/core/src/test/scala/com/salesforce/op/stages/impl/feature/RealNNVectorizerTest.scala new file mode 100644 index 0000000000..8049128dd2 --- /dev/null +++ b/core/src/test/scala/com/salesforce/op/stages/impl/feature/RealNNVectorizerTest.scala @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2017, Salesforce.com, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * * Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + + +package com.salesforce.op.stages.impl.feature + +import com.salesforce.op.features.types.{ArrayDoubleConversions, OPVector, RealNN} +import com.salesforce.op.test.{OpTransformerSpec, TestFeatureBuilder} + + +class RealNNVectorizerTest extends OpTransformerSpec[OPVector, RealNNVectorizer] { + val sample = Seq(RealNN(-1.0), RealNN(-4.0), RealNN(5.0), RealNN(-5.5), RealNN(0.1), RealNN(2.0), RealNN(0.0)) + val (inputData, f1) = TestFeatureBuilder(sample) + val transformer: RealNNVectorizer = new RealNNVectorizer().setInput(f1) + + val expectedResult: Seq[OPVector] = + Seq(Array(-1.0, -4.0, 5.0, -5.5, 0.1, 2.0, 0.0).toOPVector) + + behavior of "RealNNVectorizerTest" + + it should "transformFn" in { + + } +} From 41c513f4f5375ee59693c04b35366fd36fd64148 Mon Sep 17 00:00:00 2001 From: Chris Rupley Date: Mon, 15 Apr 2019 17:08:23 -0700 Subject: [PATCH 07/11] test for OpLDA not OpLDATest --- .../scala/com/salesforce/op/stages/impl/feature/OpLDATest.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/test/scala/com/salesforce/op/stages/impl/feature/OpLDATest.scala b/core/src/test/scala/com/salesforce/op/stages/impl/feature/OpLDATest.scala index df35c4192e..091bd316ab 100644 --- a/core/src/test/scala/com/salesforce/op/stages/impl/feature/OpLDATest.scala +++ b/core/src/test/scala/com/salesforce/op/stages/impl/feature/OpLDATest.scala @@ -78,7 +78,7 @@ class OpLDATest extends FlatSpec with TestSparkContext { .toSeq .map(_.getAs[Vector](0)) - Spec[OpLDATest] should "convert document term vectors into topic vectors" in { + Spec[OpLDA] should "convert document term vectors into topic vectors" in { val f2Vec = new OpLDA().setInput(f2).setK(k).setSeed(seed).setMaxIter(maxIter) val testTransformedData = f2Vec.fit(inputDS).transform(inputDS) val output = f2Vec.getOutput() From f9fae63fa636df5a0f04c9e012bc31ef6853b2c9 Mon Sep 17 00:00:00 2001 From: Chris Rupley Date: Tue, 16 Apr 2019 16:03:18 -0700 Subject: [PATCH 08/11] correct expected result --- .../op/stages/impl/feature/RealNNVectorizerTest.scala | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/core/src/test/scala/com/salesforce/op/stages/impl/feature/RealNNVectorizerTest.scala b/core/src/test/scala/com/salesforce/op/stages/impl/feature/RealNNVectorizerTest.scala index 8049128dd2..82899d2d3e 100644 --- a/core/src/test/scala/com/salesforce/op/stages/impl/feature/RealNNVectorizerTest.scala +++ b/core/src/test/scala/com/salesforce/op/stages/impl/feature/RealNNVectorizerTest.scala @@ -40,8 +40,7 @@ class RealNNVectorizerTest extends OpTransformerSpec[OPVector, RealNNVectorizer] val (inputData, f1) = TestFeatureBuilder(sample) val transformer: RealNNVectorizer = new RealNNVectorizer().setInput(f1) - val expectedResult: Seq[OPVector] = - Seq(Array(-1.0, -4.0, 5.0, -5.5, 0.1, 2.0, 0.0).toOPVector) + val expectedResult: Seq[OPVector] = Array(-1.0, -4.0, 5.0, -5.5, 0.1, 2.0, 0.0).map(Array(_).toOPVector) behavior of "RealNNVectorizerTest" From af4faf4cfc594f9702e5710a404825d61c70ef67 Mon Sep 17 00:00:00 2001 From: Chris Rupley Date: Tue, 16 Apr 2019 16:13:25 -0700 Subject: [PATCH 09/11] add check for usage in vectorize method --- .../op/stages/impl/feature/RealNNVectorizerTest.scala | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/core/src/test/scala/com/salesforce/op/stages/impl/feature/RealNNVectorizerTest.scala b/core/src/test/scala/com/salesforce/op/stages/impl/feature/RealNNVectorizerTest.scala index 82899d2d3e..64b71b98a3 100644 --- a/core/src/test/scala/com/salesforce/op/stages/impl/feature/RealNNVectorizerTest.scala +++ b/core/src/test/scala/com/salesforce/op/stages/impl/feature/RealNNVectorizerTest.scala @@ -44,7 +44,8 @@ class RealNNVectorizerTest extends OpTransformerSpec[OPVector, RealNNVectorizer] behavior of "RealNNVectorizerTest" - it should "transformFn" in { - + it should "be applied via shortcut" in { + val f2 = f1.vectorize() + f2.originStage.isInstanceOf[RealNNVectorizer] shouldBe true } } From b56e63621890e946e65d795a9bb6c5014e2fa38a Mon Sep 17 00:00:00 2001 From: Chris Rupley Date: Tue, 16 Apr 2019 16:20:16 -0700 Subject: [PATCH 10/11] add junit --- .../op/stages/impl/feature/RealNNVectorizerTest.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/core/src/test/scala/com/salesforce/op/stages/impl/feature/RealNNVectorizerTest.scala b/core/src/test/scala/com/salesforce/op/stages/impl/feature/RealNNVectorizerTest.scala index 64b71b98a3..c822d7c07a 100644 --- a/core/src/test/scala/com/salesforce/op/stages/impl/feature/RealNNVectorizerTest.scala +++ b/core/src/test/scala/com/salesforce/op/stages/impl/feature/RealNNVectorizerTest.scala @@ -33,8 +33,10 @@ package com.salesforce.op.stages.impl.feature import com.salesforce.op.features.types.{ArrayDoubleConversions, OPVector, RealNN} import com.salesforce.op.test.{OpTransformerSpec, TestFeatureBuilder} +import org.junit.runner.RunWith +import org.scalatest.junit.JUnitRunner - +@RunWith(classOf[JUnitRunner]) class RealNNVectorizerTest extends OpTransformerSpec[OPVector, RealNNVectorizer] { val sample = Seq(RealNN(-1.0), RealNN(-4.0), RealNN(5.0), RealNN(-5.5), RealNN(0.1), RealNN(2.0), RealNN(0.0)) val (inputData, f1) = TestFeatureBuilder(sample) From e225c39dc94b242af76a5b03a30c8368e89fb785 Mon Sep 17 00:00:00 2001 From: Chris Rupley Date: Tue, 16 Apr 2019 16:21:58 -0700 Subject: [PATCH 11/11] remove "behavior" --- .../op/stages/impl/feature/RealNNVectorizerTest.scala | 2 -- 1 file changed, 2 deletions(-) diff --git a/core/src/test/scala/com/salesforce/op/stages/impl/feature/RealNNVectorizerTest.scala b/core/src/test/scala/com/salesforce/op/stages/impl/feature/RealNNVectorizerTest.scala index c822d7c07a..b1486df14d 100644 --- a/core/src/test/scala/com/salesforce/op/stages/impl/feature/RealNNVectorizerTest.scala +++ b/core/src/test/scala/com/salesforce/op/stages/impl/feature/RealNNVectorizerTest.scala @@ -44,8 +44,6 @@ class RealNNVectorizerTest extends OpTransformerSpec[OPVector, RealNNVectorizer] val expectedResult: Seq[OPVector] = Array(-1.0, -4.0, 5.0, -5.5, 0.1, 2.0, 0.0).map(Array(_).toOPVector) - behavior of "RealNNVectorizerTest" - it should "be applied via shortcut" in { val f2 = f1.vectorize() f2.originStage.isInstanceOf[RealNNVectorizer] shouldBe true