Changes related to GC-463

- few correction based on @ezhulenev comments in the pull request
collectivemedia · Jul 29, 2015 · fae4fa1 · fae4fa1
1 parent 0c09c2e
commit fae4fa1
Show file tree

Hide file tree

Showing 26 changed files with 73 additions and 114 deletions.
diff --git a/README.md b/README.md
@@ -51,22 +51,6 @@ Schema migrations managed by [Flyway](http://flywaydb.org),
 If you want to add test that excepect modelmatrix matrix schema and tables to be present please implement trait `com.collective.modelmatrix.catalog.InstallSchemaBefore`
 
 
-<!--schema DDL and migrations located in: `modelmatrix-cli/src/main/resources/db/migration`-->
-
-<!--Install schema for development:-->
-
-<!--    sbt> project modelmatrix-cli  -->
-<!--    sbt> flywayMigrate -->
-
-<!--If you need to install schema into different database, you have to provide flyway properties at sbt startup-->
-
-<!--    sbt -Dflyway.url=myUrl \-->
-<!--        -Dflyway.user=myUser \-->
-<!--        -Dflyway.password=mySecretPwd \-->
-<!--        -Dflyway.schemas=schema1,schema2,schema3 \-->
-<!--        -Dflyway.placeholders.keyABC=valueXYZ \-->
-<!--        -Dflyway.placeholders.otherplaceholder=value123-->
-
 ## Testing
 
 Unit and Integration test are automatically creating/updating schema and using by default H2

diff --git a/build.sbt b/build.sbt
@@ -78,7 +78,6 @@ lazy val modelmatrixCore =
 lazy val modelmatrixCli =
   ModelMatrixProject("modelmatrix-cli")
     .dependsOn(modelmatrixCore)
-//    .settings(flywaySettings: _*)
 
 lazy val modelmatrixUdf =
   ModelMatrixProject("modelmatrix-udf")
diff --git a/modelmatrix-cli/src/main/scala/com/collective/modelmatrix/cli/db/InstallSchema.scala b/modelmatrix-cli/src/main/scala/com/collective/modelmatrix/cli/db/InstallSchema.scala
@@ -4,7 +4,7 @@ import com.collective.modelmatrix.cli.Script
 import com.collective.modelmatrix.db.{DefaultDBConfigWrapper, DatabaseConfigWrapper, SchemaInstaller}
 import org.slf4j.LoggerFactory
 
-case class InstallSchema() extends SchemaInstaller with Script {
+case class InstallSchema() extends Script with SchemaInstaller{
 
   private val log = LoggerFactory.getLogger(classOf[InstallSchema])
 

diff --git a/modelmatrix-cli/src/main/scala/com/collective/modelmatrix/cli/definition/AddDefinition.scala b/modelmatrix-cli/src/main/scala/com/collective/modelmatrix/cli/definition/AddDefinition.scala
@@ -3,7 +3,7 @@ package com.collective.modelmatrix.cli.definition
 import java.nio.file.Path
 import java.time.Instant
 
-import com.collective.modelmatrix.ModelMatrix.DbModelMatrixCatalog
+import com.collective.modelmatrix.ModelMatrixAccess.ModelMatrixCatalogAccess
 import com.collective.modelmatrix.cli.{ModelConfigurationParser, Script}
 import com.typesafe.config.{ConfigFactory, ConfigResolveOptions}
 import org.slf4j.LoggerFactory
@@ -15,7 +15,7 @@ case class AddDefinition(
   configPath: String,
   name: Option[String],
   comment: Option[String]
-) extends Script with DbModelMatrixCatalog {
+) extends Script with ModelMatrixCatalogAccess {
 
   private val log = LoggerFactory.getLogger(classOf[AddDefinition])
 

diff --git a/...matrix-cli/src/main/scala/com/collective/modelmatrix/cli/definition/ListDefinitions.scala b/...matrix-cli/src/main/scala/com/collective/modelmatrix/cli/definition/ListDefinitions.scala
@@ -1,12 +1,12 @@
 package com.collective.modelmatrix.cli.definition
 
-import com.collective.modelmatrix.ModelMatrix.DbModelMatrixCatalog
+import com.collective.modelmatrix.ModelMatrixAccess.ModelMatrixCatalogAccess
 import com.collective.modelmatrix.cli.Script
 import org.slf4j.LoggerFactory
 
 case class ListDefinitions(
   name: Option[String]
-) extends Script with DbModelMatrixCatalog {
+) extends Script with ModelMatrixCatalogAccess {
 
   private val log = LoggerFactory.getLogger(classOf[ListDefinitions])
 

diff --git a/modelmatrix-cli/src/main/scala/com/collective/modelmatrix/cli/definition/ViewFeatures.scala b/modelmatrix-cli/src/main/scala/com/collective/modelmatrix/cli/definition/ViewFeatures.scala
@@ -1,12 +1,12 @@
 package com.collective.modelmatrix.cli.definition
 
-import com.collective.modelmatrix.ModelMatrix.DbModelMatrixCatalog
+import com.collective.modelmatrix.ModelMatrixAccess.ModelMatrixCatalogAccess
 import com.collective.modelmatrix.cli.Script
 import org.slf4j.LoggerFactory
 
 case class ViewFeatures(
   modelDefinitionId: Int
-) extends Script with DbModelMatrixCatalog {
+) extends Script with ModelMatrixCatalogAccess {
 
   private val log = LoggerFactory.getLogger(classOf[ViewFeatures])
 

diff --git a/modelmatrix-cli/src/main/scala/com/collective/modelmatrix/cli/definition/ViewSource.scala b/modelmatrix-cli/src/main/scala/com/collective/modelmatrix/cli/definition/ViewSource.scala
@@ -1,12 +1,12 @@
 package com.collective.modelmatrix.cli.definition
 
-import com.collective.modelmatrix.ModelMatrix.DbModelMatrixCatalog
+import com.collective.modelmatrix.ModelMatrixAccess.ModelMatrixCatalogAccess
 import com.collective.modelmatrix.cli.Script
 import org.slf4j.LoggerFactory
 
 case class ViewSource(
   modelDefinitionId: Int
-) extends Script with DbModelMatrixCatalog {
+) extends Script with ModelMatrixCatalogAccess {
 
   private val log = LoggerFactory.getLogger(classOf[ViewFeatures])
 

diff --git a/...rix-cli/src/main/scala/com/collective/modelmatrix/cli/featurize/SparseFeaturization.scala b/...rix-cli/src/main/scala/com/collective/modelmatrix/cli/featurize/SparseFeaturization.scala
@@ -1,9 +1,9 @@
 package com.collective.modelmatrix.cli.featurize
 
-import com.collective.modelmatrix.ModelMatrix.DbModelMatrixCatalog
+import com.collective.modelmatrix.ModelMatrixAccess.ModelMatrixCatalogAccess
 import com.collective.modelmatrix.cli.{Source, _}
 import com.collective.modelmatrix.transform.Transformer
-import com.collective.modelmatrix.{Featurization, Labeling, ModelMatrix}
+import com.collective.modelmatrix.{Featurization, Labeling, ModelMatrixAccess}
 import org.apache.spark.mllib.linalg.{DenseVector, SparseVector}
 import org.apache.spark.sql.Row
 import org.apache.spark.sql.types._
@@ -18,7 +18,7 @@ case class SparseFeaturization(
   idColumn: String,
   repartitionSource: Option[Int],
   cacheSource: Boolean
-) extends Script with SourceTransformation with DbModelMatrixCatalog with CliSparkContext {
+) extends Script with SourceTransformation with ModelMatrixCatalogAccess with CliSparkContext {
 
   private val log = LoggerFactory.getLogger(classOf[ValidateInputData])
 
@@ -38,7 +38,7 @@ case class SparseFeaturization(
       s"Featurized sink: $sink. " +
       s"Id column: $idColumn")
 
-    implicit val sqlContext = ModelMatrix.hiveContext(sc)
+    implicit val sqlContext = ModelMatrixAccess.hiveContext(sc)
 
     val features = blockOn(db.run(modelInstanceFeatures.features(modelInstanceId)))
     require(features.nonEmpty, s"No features are defined for model instance: $modelInstanceId. " +

diff --git a/...atrix-cli/src/main/scala/com/collective/modelmatrix/cli/featurize/ValidateInputData.scala b/...atrix-cli/src/main/scala/com/collective/modelmatrix/cli/featurize/ValidateInputData.scala
@@ -1,9 +1,9 @@
 package com.collective.modelmatrix.cli.featurize
 
-import com.collective.modelmatrix.ModelMatrix.DbModelMatrixCatalog
+import com.collective.modelmatrix.ModelMatrixAccess.ModelMatrixCatalogAccess
 import com.collective.modelmatrix.cli.{SourceTransformation, CliSparkContext, Script, Source}
 import com.collective.modelmatrix.transform.Transformer
-import com.collective.modelmatrix.{Featurization, ModelMatrix}
+import com.collective.modelmatrix.{Featurization, ModelMatrixAccess}
 import org.slf4j.LoggerFactory
 
 import scalaz._
@@ -13,7 +13,7 @@ case class ValidateInputData(
   source: Source,
   repartitionSource: Option[Int],
   cacheSource: Boolean
-) extends Script with SourceTransformation with DbModelMatrixCatalog with CliSparkContext {
+) extends Script with SourceTransformation with ModelMatrixCatalogAccess with CliSparkContext {
 
   private val log = LoggerFactory.getLogger(classOf[ValidateInputData])
 
@@ -24,7 +24,7 @@ case class ValidateInputData(
     log.info(s"Validate input data against Model Matrix instance: $modelInstanceId. " +
       s"Data source: $source")
 
-    implicit val sqlContext = ModelMatrix.hiveContext(sc)
+    implicit val sqlContext = ModelMatrixAccess.hiveContext(sc)
 
     val features = blockOn(db.run(modelInstanceFeatures.features(modelInstanceId)))
     require(features.nonEmpty, s"No features are defined for model instance: $modelInstanceId. " +

diff --git a/modelmatrix-cli/src/main/scala/com/collective/modelmatrix/cli/instance/AddInstance.scala b/modelmatrix-cli/src/main/scala/com/collective/modelmatrix/cli/instance/AddInstance.scala
@@ -1,6 +1,6 @@
 package com.collective.modelmatrix.cli.instance
 
-import com.collective.modelmatrix.ModelMatrix.DbModelMatrixCatalog
+import com.collective.modelmatrix.ModelMatrixAccess.ModelMatrixCatalogAccess
 import com.collective.modelmatrix._
 import com.collective.modelmatrix.cli.{SourceTransformation, CliSparkContext, Script, Source}
 import com.collective.modelmatrix.transform._
@@ -19,14 +19,14 @@ case class AddInstance(
   cacheSource: Boolean
 )
   extends Script with SourceTransformation
-  with DbModelMatrixCatalog
+  with ModelMatrixCatalogAccess
   with CliSparkContext
   with Transformers
   with TransformationProcess {
 
   private val log = LoggerFactory.getLogger(classOf[AddInstance])
 
-  private implicit lazy val sqlContext = ModelMatrix.hiveContext(sc)
+  private implicit lazy val sqlContext = ModelMatrixAccess.hiveContext(sc)
 
   import com.collective.modelmatrix.cli.ASCIITableFormat._
   import com.collective.modelmatrix.cli.ASCIITableFormats._

diff --git a/modelmatrix-cli/src/main/scala/com/collective/modelmatrix/cli/instance/ListInstances.scala b/modelmatrix-cli/src/main/scala/com/collective/modelmatrix/cli/instance/ListInstances.scala
@@ -1,13 +1,13 @@
 package com.collective.modelmatrix.cli.instance
 
-import com.collective.modelmatrix.ModelMatrix.DbModelMatrixCatalog
+import com.collective.modelmatrix.ModelMatrixAccess.ModelMatrixCatalogAccess
 import com.collective.modelmatrix.cli.Script
 import org.slf4j.LoggerFactory
 
 
 case class ListInstances(
   modelDefinitionId: Option[Int], name: Option[String]
-) extends Script with DbModelMatrixCatalog {
+) extends Script with ModelMatrixCatalogAccess {
 
   private val log = LoggerFactory.getLogger(classOf[ListInstances])
 

diff --git a/...matrix-cli/src/main/scala/com/collective/modelmatrix/cli/instance/ValidateInputData.scala b/...matrix-cli/src/main/scala/com/collective/modelmatrix/cli/instance/ValidateInputData.scala
@@ -1,7 +1,7 @@
 package com.collective.modelmatrix.cli.instance
 
-import com.collective.modelmatrix.ModelMatrix
-import com.collective.modelmatrix.ModelMatrix.DbModelMatrixCatalog
+import com.collective.modelmatrix.ModelMatrixAccess
+import com.collective.modelmatrix.ModelMatrixAccess.ModelMatrixCatalogAccess
 import com.collective.modelmatrix.cli.{Source, _}
 import com.collective.modelmatrix.transform._
 import org.slf4j.LoggerFactory
@@ -13,11 +13,11 @@ case class ValidateInputData(
   source: Source,
   repartitionSource: Option[Int],
   cacheSource: Boolean
-) extends Script with SourceTransformation with DbModelMatrixCatalog with CliSparkContext with Transformers {
+) extends Script with SourceTransformation with ModelMatrixCatalogAccess with CliSparkContext with Transformers {
 
   private val log = LoggerFactory.getLogger(classOf[ValidateInputData])
 
-  private implicit lazy val sqlContext = ModelMatrix.hiveContext(sc)
+  private implicit lazy val sqlContext = ModelMatrixAccess.hiveContext(sc)
 
   import com.collective.modelmatrix.cli.ASCIITableFormat._
   import com.collective.modelmatrix.cli.ASCIITableFormats._

diff --git a/modelmatrix-cli/src/main/scala/com/collective/modelmatrix/cli/instance/ViewColumns.scala b/modelmatrix-cli/src/main/scala/com/collective/modelmatrix/cli/instance/ViewColumns.scala
@@ -1,6 +1,6 @@
 package com.collective.modelmatrix.cli.instance
 
-import com.collective.modelmatrix.ModelMatrix.DbModelMatrixCatalog
+import com.collective.modelmatrix.ModelMatrixAccess.ModelMatrixCatalogAccess
 import com.collective.modelmatrix.catalog._
 import com.collective.modelmatrix.cli._
 import com.collective.modelmatrix.{CategoricalColumn, BinColumn}
@@ -11,7 +11,7 @@ import scalaz._
 
 case class ViewColumns(
   modelInstanceId: Int, group: Option[String], feature: Option[String]
-) extends Script with DbModelMatrixCatalog {
+) extends Script with ModelMatrixCatalogAccess {
 
   private val log = LoggerFactory.getLogger(classOf[ViewColumns])
 

diff --git a/modelmatrix-cli/src/main/scala/com/collective/modelmatrix/cli/instance/ViewFeatures.scala b/modelmatrix-cli/src/main/scala/com/collective/modelmatrix/cli/instance/ViewFeatures.scala
@@ -1,13 +1,13 @@
 package com.collective.modelmatrix.cli.instance
 
-import com.collective.modelmatrix.ModelMatrix.DbModelMatrixCatalog
+import com.collective.modelmatrix.ModelMatrixAccess.ModelMatrixCatalogAccess
 import com.collective.modelmatrix.cli.Script
 import org.slf4j.LoggerFactory
 
 
 case class ViewFeatures(
   modelInstanceId: Int
-) extends Script with DbModelMatrixCatalog {
+) extends Script with ModelMatrixCatalogAccess {
 
   private val log = LoggerFactory.getLogger(classOf[ViewFeatures])
 

diff --git a/modelmatrix-core/src/it/resources/pg.conf b/modelmatrix-core/src/it/resources/pg.conf
diff --git a/modelmatrix-core/src/it/scala/com/collective/modelmatrix/FeaturizationSpec.scala b/modelmatrix-core/src/it/scala/com/collective/modelmatrix/FeaturizationSpec.scala
@@ -12,7 +12,7 @@ import scalaz.{-\/, \/, \/-}
 
 class FeaturizationSpec extends FlatSpec with GivenWhenThen with TestSparkContext {
 
-  val sqlContext = ModelMatrix.sqlContext(sc)
+  val sqlContext = ModelMatrixAccess.sqlContext(sc)
 
   val schema = StructType(Seq(
     StructField("auction_id", LongType),

diff --git a/modelmatrix-core/src/it/scala/com/collective/modelmatrix/transform/BinsTransformerSpec.scala b/modelmatrix-core/src/it/scala/com/collective/modelmatrix/transform/BinsTransformerSpec.scala
@@ -1,17 +1,17 @@
 package com.collective.modelmatrix.transform
 
-import com.collective.modelmatrix.{ModelMatrix, ModelFeature, TestSparkContext}
-import org.apache.spark.sql.{Row, SQLContext}
+import com.collective.modelmatrix.{ModelFeature, ModelMatrixAccess, TestSparkContext}
+import org.apache.spark.sql.Row
 import org.apache.spark.sql.types._
 import org.scalatest.FlatSpec
 
 import scala.util.Random
-import scalaz.{\/-, -\/}
 import scalaz.syntax.either._
+import scalaz.{-\/, \/-}
 
 class BinsTransformerSpec extends FlatSpec with TestSparkContext {
 
-  val sqlContext = ModelMatrix.sqlContext(sc)
+  val sqlContext = ModelMatrixAccess.sqlContext(sc)
 
   val schema = StructType(Seq(
     StructField("adv_site", StringType),

diff --git a/...atrix-core/src/it/scala/com/collective/modelmatrix/transform/ExtractExpressionsSpec.scala b/...atrix-core/src/it/scala/com/collective/modelmatrix/transform/ExtractExpressionsSpec.scala
@@ -5,7 +5,7 @@ import java.time.{DayOfWeek, Instant, ZoneOffset}
 import java.util.UUID
 
 import com.collective.modelmatrix.CategoricalColumn.CategoricalValue
-import com.collective.modelmatrix.{ModelMatrixEncoding, ModelFeature, ModelMatrix, TestSparkContext}
+import com.collective.modelmatrix.{ModelMatrixEncoding, ModelFeature, ModelMatrixAccess, TestSparkContext}
 import org.apache.spark.sql.Row
 import org.apache.spark.sql.types._
 import org.scalatest.FlatSpec
@@ -17,7 +17,7 @@ import scalaz.syntax.either._
 
 class ExtractExpressionsSpec extends FlatSpec with TestSparkContext {
 
-  val sqlContext = ModelMatrix.sqlContext(sc)
+  val sqlContext = ModelMatrixAccess.sqlContext(sc)
 
   val rnd = new Random()
 

diff --git a/...trix-core/src/it/scala/com/collective/modelmatrix/transform/IdentityTransformerSpec.scala b/...trix-core/src/it/scala/com/collective/modelmatrix/transform/IdentityTransformerSpec.scala
@@ -1,6 +1,6 @@
 package com.collective.modelmatrix.transform
 
-import com.collective.modelmatrix.{ModelMatrix, ModelFeature, TestSparkContext}
+import com.collective.modelmatrix.{ModelMatrixAccess, ModelFeature, TestSparkContext}
 import org.apache.spark.sql.types.{IntegerType, StringType, StructField, StructType}
 import org.apache.spark.sql.{Row, SQLContext}
 import org.scalatest.FlatSpec
@@ -10,7 +10,7 @@ import scalaz.syntax.either._
 
 class IdentityTransformerSpec extends FlatSpec with TestSparkContext {
 
-  val sqlContext = ModelMatrix.sqlContext(sc)
+  val sqlContext = ModelMatrixAccess.sqlContext(sc)
 
   val schema = StructType(Seq(
     StructField("adv_site", StringType),

diff --git a/...lmatrix-core/src/it/scala/com/collective/modelmatrix/transform/IndexTransformerSpec.scala b/...lmatrix-core/src/it/scala/com/collective/modelmatrix/transform/IndexTransformerSpec.scala
@@ -1,7 +1,7 @@
 package com.collective.modelmatrix.transform
 
 import com.collective.modelmatrix.CategoricalColumn.{AllOther, CategoricalValue}
-import com.collective.modelmatrix.{ModelMatrixEncoding, ModelMatrix, ModelFeature, TestSparkContext}
+import com.collective.modelmatrix.{ModelMatrixEncoding, ModelMatrixAccess, ModelFeature, TestSparkContext}
 import org.apache.spark.sql.{Row, SQLContext}
 import org.apache.spark.sql.types.{StringType, StructField, StructType}
 import org.scalatest.FlatSpec
@@ -13,7 +13,7 @@ import scalaz.syntax.either._
 
 class IndexTransformerSpec extends FlatSpec with TestSparkContext {
 
-  val sqlContext = ModelMatrix.sqlContext(sc)
+  val sqlContext = ModelMatrixAccess.sqlContext(sc)
 
   val schema = StructType(Seq(
     StructField("adv_site", StringType)

diff --git a/modelmatrix-core/src/it/scala/com/collective/modelmatrix/transform/TopTransformerSpec.scala b/modelmatrix-core/src/it/scala/com/collective/modelmatrix/transform/TopTransformerSpec.scala
@@ -1,7 +1,7 @@
 package com.collective.modelmatrix.transform
 
 import com.collective.modelmatrix.CategoricalColumn.{AllOther, CategoricalValue}
-import com.collective.modelmatrix.{ModelMatrixEncoding, ModelMatrix, ModelFeature, TestSparkContext}
+import com.collective.modelmatrix.{ModelMatrixEncoding, ModelMatrixAccess, ModelFeature, TestSparkContext}
 import org.apache.spark.sql.{DataFrame, Row, SQLContext}
 import org.apache.spark.sql.types._
 import org.scalatest.FlatSpec
@@ -11,7 +11,7 @@ import scalaz.syntax.either._
 
 class TopTransformerSpec extends FlatSpec with TestSparkContext {
 
-  val sqlContext = ModelMatrix.sqlContext(sc)
+  val sqlContext = ModelMatrixAccess.sqlContext(sc)
 
   val schema = StructType(Seq(
     StructField("adv_site", StringType)

diff --git a/modelmatrix-core/src/it/scala/com/collective/modelmatrix/transform/TransformerSpec.scala b/modelmatrix-core/src/it/scala/com/collective/modelmatrix/transform/TransformerSpec.scala
@@ -1,6 +1,6 @@
 package com.collective.modelmatrix.transform
 
-import com.collective.modelmatrix.{ModelFeature, ModelMatrix, TestSparkContext}
+import com.collective.modelmatrix.{ModelFeature, ModelMatrixAccess, TestSparkContext}
 import org.apache.spark.sql.Row
 import org.apache.spark.sql.types.{IntegerType, StringType, StructField, StructType}
 import org.scalatest.FlatSpec
@@ -9,7 +9,7 @@ import scalaz.{\/-, -\/}
 
 class TransformerSpec extends FlatSpec with TestSparkContext {
 
-  val sqlContext = ModelMatrix.sqlContext(sc)
+  val sqlContext = ModelMatrixAccess.sqlContext(sc)
 
   val schema = StructType(Seq(
     StructField("adv_site", StringType),