diff --git a/core/src/main/scala/ai/h2o/sparkling/frame/H2OColumn.scala b/core/src/main/scala/ai/h2o/sparkling/H2OColumn.scala similarity index 97% rename from core/src/main/scala/ai/h2o/sparkling/frame/H2OColumn.scala rename to core/src/main/scala/ai/h2o/sparkling/H2OColumn.scala index bb66e5c59c..0e03c13fa1 100644 --- a/core/src/main/scala/ai/h2o/sparkling/frame/H2OColumn.scala +++ b/core/src/main/scala/ai/h2o/sparkling/H2OColumn.scala @@ -15,7 +15,7 @@ * limitations under the License. */ -package ai.h2o.sparkling.frame +package ai.h2o.sparkling case class H2OColumn( name: String, diff --git a/core/src/main/scala/ai/h2o/sparkling/frame/H2OColumnType.scala b/core/src/main/scala/ai/h2o/sparkling/H2OColumnType.scala similarity index 97% rename from core/src/main/scala/ai/h2o/sparkling/frame/H2OColumnType.scala rename to core/src/main/scala/ai/h2o/sparkling/H2OColumnType.scala index b3669a05e2..f72198d5d4 100644 --- a/core/src/main/scala/ai/h2o/sparkling/frame/H2OColumnType.scala +++ b/core/src/main/scala/ai/h2o/sparkling/H2OColumnType.scala @@ -15,7 +15,7 @@ * limitations under the License. */ -package ai.h2o.sparkling.frame +package ai.h2o.sparkling object H2OColumnType extends Enumeration { val enum, string, int, real, time, uuid = Value diff --git a/core/src/main/scala/ai/h2o/sparkling/frame/H2OFrame.scala b/core/src/main/scala/ai/h2o/sparkling/H2OFrame.scala similarity index 97% rename from core/src/main/scala/ai/h2o/sparkling/frame/H2OFrame.scala rename to core/src/main/scala/ai/h2o/sparkling/H2OFrame.scala index 2cd6b15d07..fb3e32e997 100644 --- a/core/src/main/scala/ai/h2o/sparkling/frame/H2OFrame.scala +++ b/core/src/main/scala/ai/h2o/sparkling/H2OFrame.scala @@ -15,16 +15,15 @@ * limitations under the License. */ -package ai.h2o.sparkling.frame +package ai.h2o.sparkling import java.text.MessageFormat -import ai.h2o.sparkling.backend.NodeDesc import ai.h2o.sparkling.backend.utils.RestApiUtils._ import ai.h2o.sparkling.backend.utils.{RestCommunication, RestEncodingUtils} +import ai.h2o.sparkling.backend.{H2OChunk, H2OJob, NodeDesc} import ai.h2o.sparkling.extensions.rest.api.Paths import ai.h2o.sparkling.extensions.rest.api.schema.{FinalizeFrameV3, InitializeFrameV3} -import ai.h2o.sparkling.job.H2OJob import ai.h2o.sparkling.utils.Base64Encoding import org.apache.spark.h2o.{H2OConf, H2OContext} import water.api.schemas3.FrameChunksV3.FrameChunkV3 @@ -34,7 +33,7 @@ import water.api.schemas3._ /** * H2OFrame representation via Rest API */ -class H2OFrame private(val frameId: String, val columns: Array[H2OColumn], val chunks: Array[H2OChunk]) +class H2OFrame private(val frameId: String, val columns: Array[H2OColumn], private[sparkling] val chunks: Array[H2OChunk]) extends Serializable with RestEncodingUtils { private val conf = H2OContext.ensure("H2OContext needs to be running in order to create H2OFrame").getConf val columnNames: Array[String] = columns.map(_.name) diff --git a/core/src/main/scala/ai/h2o/sparkling/frame/H2OChunk.scala b/core/src/main/scala/ai/h2o/sparkling/backend/H2OChunk.scala similarity index 94% rename from core/src/main/scala/ai/h2o/sparkling/frame/H2OChunk.scala rename to core/src/main/scala/ai/h2o/sparkling/backend/H2OChunk.scala index 994b4bd170..8583372860 100644 --- a/core/src/main/scala/ai/h2o/sparkling/frame/H2OChunk.scala +++ b/core/src/main/scala/ai/h2o/sparkling/backend/H2OChunk.scala @@ -15,20 +15,19 @@ * limitations under the License. */ -package ai.h2o.sparkling.frame +package ai.h2o.sparkling.backend import java.io.{InputStream, OutputStream} -import ai.h2o.sparkling.backend.NodeDesc import ai.h2o.sparkling.backend.utils.{RestApiUtils, RestCommunication} import ai.h2o.sparkling.extensions.rest.api.Paths import ai.h2o.sparkling.utils.{Base64Encoding, Compression} import org.apache.spark.h2o.H2OConf -case class H2OChunk(index: Int, numberOfRows: Int, location: NodeDesc) +private[sparkling] case class H2OChunk(index: Int, numberOfRows: Int, location: NodeDesc) -object H2OChunk extends RestCommunication { +private[sparkling] object H2OChunk extends RestCommunication { def getChunkAsInputStream( node: NodeDesc, conf: H2OConf, diff --git a/core/src/main/scala/ai/h2o/sparkling/backend/H2ODataFrame.scala b/core/src/main/scala/ai/h2o/sparkling/backend/H2ODataFrame.scala index fe418e3a18..289687e8c5 100644 --- a/core/src/main/scala/ai/h2o/sparkling/backend/H2ODataFrame.scala +++ b/core/src/main/scala/ai/h2o/sparkling/backend/H2ODataFrame.scala @@ -17,10 +17,9 @@ package ai.h2o.sparkling.backend -import ai.h2o.sparkling.SparkTimeZone import ai.h2o.sparkling.backend.utils.SupportedTypes._ import ai.h2o.sparkling.backend.utils.{ConversionUtils, ReflectionUtils} -import ai.h2o.sparkling.frame.H2OFrame +import ai.h2o.sparkling.{H2OFrame, SparkTimeZone} import org.apache.spark.h2o.H2OContext import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.types.DataType diff --git a/core/src/main/scala/ai/h2o/sparkling/backend/H2OFrameRelation.scala b/core/src/main/scala/ai/h2o/sparkling/backend/H2OFrameRelation.scala index 9ba4a77036..a426c80757 100644 --- a/core/src/main/scala/ai/h2o/sparkling/backend/H2OFrameRelation.scala +++ b/core/src/main/scala/ai/h2o/sparkling/backend/H2OFrameRelation.scala @@ -18,7 +18,7 @@ package ai.h2o.sparkling.backend import ai.h2o.sparkling.backend.utils.ReflectionUtils -import ai.h2o.sparkling.frame.{H2OColumn, H2OColumnType, H2OFrame} +import ai.h2o.sparkling.{H2OColumn, H2OColumnType, H2OFrame} import org.apache.spark.h2o.H2OContext import org.apache.spark.rdd.RDD import org.apache.spark.sql.sources.{BaseRelation, PrunedScan, TableScan} diff --git a/core/src/main/scala/ai/h2o/sparkling/job/H2OJob.scala b/core/src/main/scala/ai/h2o/sparkling/backend/H2OJob.scala similarity index 93% rename from core/src/main/scala/ai/h2o/sparkling/job/H2OJob.scala rename to core/src/main/scala/ai/h2o/sparkling/backend/H2OJob.scala index fda1700592..f1e933c3f5 100644 --- a/core/src/main/scala/ai/h2o/sparkling/job/H2OJob.scala +++ b/core/src/main/scala/ai/h2o/sparkling/backend/H2OJob.scala @@ -15,7 +15,7 @@ * limitations under the License. */ -package ai.h2o.sparkling.job +package ai.h2o.sparkling.backend import ai.h2o.sparkling.backend.utils.RestApiUtils.getClusterEndpoint import ai.h2o.sparkling.backend.utils.RestCommunication @@ -23,7 +23,7 @@ import org.apache.spark.expose.Logging import org.apache.spark.h2o.{H2OConf, H2OContext} import water.api.schemas3.{JobV3, JobsV3} -class H2OJob private(val id: String) extends Logging { +private[sparkling] class H2OJob private(val id: String) extends Logging { private val conf = H2OContext.ensure("H2OContext needs to be running!").getConf private def fetchStatus(): H2OJobStatus.Value = { @@ -48,7 +48,7 @@ class H2OJob private(val id: String) extends Logging { } } -object H2OJob extends RestCommunication { +private[sparkling] object H2OJob extends RestCommunication { def apply(jobId: String): H2OJob = { val conf = H2OContext.ensure().getConf verifyAndGetJob(conf, jobId) diff --git a/core/src/main/scala/ai/h2o/sparkling/job/H2OJobStatus.scala b/core/src/main/scala/ai/h2o/sparkling/backend/H2OJobStatus.scala similarity index 96% rename from core/src/main/scala/ai/h2o/sparkling/job/H2OJobStatus.scala rename to core/src/main/scala/ai/h2o/sparkling/backend/H2OJobStatus.scala index d610c12dfb..6953d889db 100644 --- a/core/src/main/scala/ai/h2o/sparkling/job/H2OJobStatus.scala +++ b/core/src/main/scala/ai/h2o/sparkling/backend/H2OJobStatus.scala @@ -15,7 +15,7 @@ * limitations under the License. */ -package ai.h2o.sparkling.job +package ai.h2o.sparkling.backend object H2OJobStatus extends Enumeration { val DONE, CANCELLED, FAILED, RUNNING = Value diff --git a/core/src/main/scala/ai/h2o/sparkling/backend/H2ORDD.scala b/core/src/main/scala/ai/h2o/sparkling/backend/H2ORDD.scala index 45d83b542c..31bdec02fc 100644 --- a/core/src/main/scala/ai/h2o/sparkling/backend/H2ORDD.scala +++ b/core/src/main/scala/ai/h2o/sparkling/backend/H2ORDD.scala @@ -19,9 +19,8 @@ package ai.h2o.sparkling.backend import java.lang.reflect.Constructor -import ai.h2o.sparkling.SparkTimeZone import ai.h2o.sparkling.backend.utils.{ConversionUtils, ProductType} -import ai.h2o.sparkling.frame.H2OFrame +import ai.h2o.sparkling.{H2OFrame, SparkTimeZone} import org.apache.spark.h2o.H2OContext import org.apache.spark.{Partition, TaskContext} diff --git a/core/src/main/scala/ai/h2o/sparkling/backend/H2OSparkEntity.scala b/core/src/main/scala/ai/h2o/sparkling/backend/H2OSparkEntity.scala index 803f319f92..b71fe02b63 100644 --- a/core/src/main/scala/ai/h2o/sparkling/backend/H2OSparkEntity.scala +++ b/core/src/main/scala/ai/h2o/sparkling/backend/H2OSparkEntity.scala @@ -17,7 +17,7 @@ package ai.h2o.sparkling.backend -import ai.h2o.sparkling.frame.H2OFrame +import ai.h2o.sparkling.H2OFrame import org.apache.spark.Partition /** diff --git a/core/src/main/scala/ai/h2o/sparkling/backend/Reader.scala b/core/src/main/scala/ai/h2o/sparkling/backend/Reader.scala index cc03deaae0..3c5234d56c 100644 --- a/core/src/main/scala/ai/h2o/sparkling/backend/Reader.scala +++ b/core/src/main/scala/ai/h2o/sparkling/backend/Reader.scala @@ -23,7 +23,6 @@ import ai.h2o.sparkling.backend.converters.TimeZoneConversions import ai.h2o.sparkling.backend.utils.SupportedTypes import ai.h2o.sparkling.backend.utils.SupportedTypes._ import ai.h2o.sparkling.extensions.serde.ChunkAutoBufferReader -import ai.h2o.sparkling.frame.H2OChunk import org.apache.spark.h2o.H2OConf import org.apache.spark.unsafe.types.UTF8String diff --git a/core/src/main/scala/ai/h2o/sparkling/backend/Writer.scala b/core/src/main/scala/ai/h2o/sparkling/backend/Writer.scala index d2d6354596..97219b3680 100644 --- a/core/src/main/scala/ai/h2o/sparkling/backend/Writer.scala +++ b/core/src/main/scala/ai/h2o/sparkling/backend/Writer.scala @@ -20,9 +20,9 @@ package ai.h2o.sparkling.backend import java.io.Closeable import java.util.TimeZone +import ai.h2o.sparkling.H2OFrame import ai.h2o.sparkling.backend.converters.TimeZoneConverter import ai.h2o.sparkling.extensions.serde.{ChunkAutoBufferWriter, SerdeUtils} -import ai.h2o.sparkling.frame.{H2OChunk, H2OFrame} import ai.h2o.sparkling.utils.ScalaUtils.withResource import ai.h2o.sparkling.utils.SparkSessionUtils import org.apache.spark.h2o.{H2OContext, RDD} diff --git a/core/src/main/scala/ai/h2o/sparkling/backend/converters/SparkDataFrameConverter.scala b/core/src/main/scala/ai/h2o/sparkling/backend/converters/SparkDataFrameConverter.scala index e49346641e..701c12a885 100644 --- a/core/src/main/scala/ai/h2o/sparkling/backend/converters/SparkDataFrameConverter.scala +++ b/core/src/main/scala/ai/h2o/sparkling/backend/converters/SparkDataFrameConverter.scala @@ -47,7 +47,7 @@ object SparkDataFrameConverter extends Logging { def toDataFrame[T <: Frame](hc: H2OContext, fr: T, copyMetadata: Boolean): DataFrame = { DKV.put(fr) - toDataFrame(hc, ai.h2o.sparkling.frame.H2OFrame(fr._key.toString), copyMetadata) + toDataFrame(hc, ai.h2o.sparkling.H2OFrame(fr._key.toString), copyMetadata) } /** @@ -59,7 +59,7 @@ object SparkDataFrameConverter extends Logging { * @return a new DataFrame definition using given H2OFrame as data source */ - def toDataFrame(hc: H2OContext, fr: ai.h2o.sparkling.frame.H2OFrame, copyMetadata: Boolean): DataFrame = { + def toDataFrame(hc: H2OContext, fr: ai.h2o.sparkling.H2OFrame, copyMetadata: Boolean): DataFrame = { val spark = SparkSessionUtils.active val relation = H2OFrameRelation(fr, copyMetadata)(spark.sqlContext) spark.baseRelationToDataFrame(relation) diff --git a/core/src/main/scala/ai/h2o/sparkling/backend/converters/SupportedRDDConverter.scala b/core/src/main/scala/ai/h2o/sparkling/backend/converters/SupportedRDDConverter.scala index f2642ef7d1..495a6f4bcd 100644 --- a/core/src/main/scala/ai/h2o/sparkling/backend/converters/SupportedRDDConverter.scala +++ b/core/src/main/scala/ai/h2o/sparkling/backend/converters/SupportedRDDConverter.scala @@ -43,11 +43,11 @@ object SupportedRDDConverter { /** Transform H2OFrame to RDD */ def toRDD[A <: Product : TypeTag : ClassTag, T <: Frame](hc: H2OContext, fr: T): RDD[A] = { DKV.put(fr) - toRDD(hc, ai.h2o.sparkling.frame.H2OFrame(fr._key.toString)) + toRDD(hc, ai.h2o.sparkling.H2OFrame(fr._key.toString)) } /** Transform H2OFrame to RDD */ - def toRDD[A <: Product : TypeTag : ClassTag](hc: H2OContext, fr: ai.h2o.sparkling.frame.H2OFrame): RDD[A] = { + def toRDD[A <: Product : TypeTag : ClassTag](hc: H2OContext, fr: ai.h2o.sparkling.H2OFrame): RDD[A] = { new H2ORDD[A](fr)(hc) } } diff --git a/core/src/main/scala/ai/h2o/sparkling/backend/utils/ReflectionUtils.scala b/core/src/main/scala/ai/h2o/sparkling/backend/utils/ReflectionUtils.scala index 7fe6a1b669..5675b1668d 100644 --- a/core/src/main/scala/ai/h2o/sparkling/backend/utils/ReflectionUtils.scala +++ b/core/src/main/scala/ai/h2o/sparkling/backend/utils/ReflectionUtils.scala @@ -18,7 +18,7 @@ package ai.h2o.sparkling.backend.utils import ai.h2o.sparkling.backend.utils.SupportedTypes._ -import ai.h2o.sparkling.frame.{H2OColumn, H2OColumnType} +import ai.h2o.sparkling.{H2OColumn, H2OColumnType} import org.apache.spark.sql.types._ import water.api.API import water.fvec.Vec diff --git a/core/src/main/scala/org/apache/spark/h2o/DefaultSource.scala b/core/src/main/scala/org/apache/spark/h2o/DefaultSource.scala index 95c24d7761..cc9cc6dd1c 100644 --- a/core/src/main/scala/org/apache/spark/h2o/DefaultSource.scala +++ b/core/src/main/scala/org/apache/spark/h2o/DefaultSource.scala @@ -62,7 +62,7 @@ class DefaultSource extends RelationProvider schema: StructType): H2OFrameRelation = { val key = checkKey(parameters) - H2OFrameRelation(ai.h2o.sparkling.frame.H2OFrame(key), copyMetadata = true)(sqlContext) + H2OFrameRelation(ai.h2o.sparkling.H2OFrame(key), copyMetadata = true)(sqlContext) } override def createRelation(sqlContext: SQLContext, diff --git a/core/src/main/scala/org/apache/spark/h2o/H2OContext.scala b/core/src/main/scala/org/apache/spark/h2o/H2OContext.scala index 47be2f2bdb..832d68c37c 100644 --- a/core/src/main/scala/org/apache/spark/h2o/H2OContext.scala +++ b/core/src/main/scala/org/apache/spark/h2o/H2OContext.scala @@ -235,7 +235,7 @@ class H2OContext private(private val conf: H2OConf) extends H2OContextExtensions def apply[T <: Frame](fr: T): RDD[A] = SupportedRDDConverter.toRDD[A, T](H2OContext.this, fr) } - def asRDD[A <: Product : TypeTag : ClassTag](fr: ai.h2o.sparkling.frame.H2OFrame): org.apache.spark.rdd.RDD[A] = { + def asRDD[A <: Product : TypeTag : ClassTag](fr: ai.h2o.sparkling.H2OFrame): org.apache.spark.rdd.RDD[A] = { SupportedRDDConverter.toRDD[A](this, fr) } @@ -246,7 +246,7 @@ class H2OContext private(private val conf: H2OConf) extends H2OContextExtensions } def asDataFrame(s: String, copyMetadata: Boolean): DataFrame = { - val frame = ai.h2o.sparkling.frame.H2OFrame(s) + val frame = ai.h2o.sparkling.H2OFrame(s) SparkDataFrameConverter.toDataFrame(this, frame, copyMetadata) } diff --git a/core/src/test/scala/ai/h2o/sparkling/frame/H2OFrameTestSuite.scala b/core/src/test/scala/ai/h2o/sparkling/frame/H2OFrameTestSuite.scala index ea4215bdef..3f9a88fa88 100644 --- a/core/src/test/scala/ai/h2o/sparkling/frame/H2OFrameTestSuite.scala +++ b/core/src/test/scala/ai/h2o/sparkling/frame/H2OFrameTestSuite.scala @@ -16,6 +16,7 @@ */ package ai.h2o.sparkling.frame +import ai.h2o.sparkling.{H2OColumnType, H2OFrame} import org.apache.spark.SparkContext import org.apache.spark.h2o.utils.SharedH2OTestContext import org.junit.runner.RunWith diff --git a/ml/src/main/scala/ai/h2o/sparkling/ml/algos/H2OAlgoCommonUtils.scala b/ml/src/main/scala/ai/h2o/sparkling/ml/algos/H2OAlgoCommonUtils.scala index 88c37e3dbc..63312dc42c 100644 --- a/ml/src/main/scala/ai/h2o/sparkling/ml/algos/H2OAlgoCommonUtils.scala +++ b/ml/src/main/scala/ai/h2o/sparkling/ml/algos/H2OAlgoCommonUtils.scala @@ -16,7 +16,7 @@ */ package ai.h2o.sparkling.ml.algos -import ai.h2o.sparkling.frame.H2OFrame +import ai.h2o.sparkling.H2OFrame import ai.h2o.sparkling.ml.params.H2OCommonParams import ai.h2o.sparkling.ml.utils.{EstimatorCommonUtils, SchemaUtils} import org.apache.spark.h2o.H2OContext diff --git a/ml/src/main/scala/ai/h2o/sparkling/ml/algos/H2OAlgorithm.scala b/ml/src/main/scala/ai/h2o/sparkling/ml/algos/H2OAlgorithm.scala index 6f73ea78fe..a2a0c81d01 100644 --- a/ml/src/main/scala/ai/h2o/sparkling/ml/algos/H2OAlgorithm.scala +++ b/ml/src/main/scala/ai/h2o/sparkling/ml/algos/H2OAlgorithm.scala @@ -16,12 +16,12 @@ */ package ai.h2o.sparkling.ml.algos +import ai.h2o.sparkling.H2OFrame import ai.h2o.sparkling.backend.exceptions.RestApiCommunicationException import ai.h2o.sparkling.backend.utils.RestCommunication -import ai.h2o.sparkling.frame.H2OFrame +import ai.h2o.sparkling.ml.internals.H2OModel import ai.h2o.sparkling.ml.models.{H2OMOJOModel, H2OMOJOSettings} import ai.h2o.sparkling.ml.params.H2OAlgoCommonParams -import ai.h2o.sparkling.model.H2OModel import hex.Model import org.apache.spark.annotation.DeveloperApi import org.apache.spark.ml.Estimator diff --git a/ml/src/main/scala/ai/h2o/sparkling/ml/algos/H2OAutoML.scala b/ml/src/main/scala/ai/h2o/sparkling/ml/algos/H2OAutoML.scala index 5d0b926679..6412818a8f 100644 --- a/ml/src/main/scala/ai/h2o/sparkling/ml/algos/H2OAutoML.scala +++ b/ml/src/main/scala/ai/h2o/sparkling/ml/algos/H2OAutoML.scala @@ -16,12 +16,12 @@ */ package ai.h2o.sparkling.ml.algos +import ai.h2o.sparkling.H2OFrame import ai.h2o.sparkling.backend.utils.{RestApiUtils, RestCommunication} -import ai.h2o.sparkling.frame.H2OFrame +import ai.h2o.sparkling.ml.internals.H2OModel import ai.h2o.sparkling.ml.models.{H2OMOJOModel, H2OMOJOSettings} import ai.h2o.sparkling.ml.params._ import ai.h2o.sparkling.ml.utils.H2OParamsReadable -import ai.h2o.sparkling.model.H2OModel import ai.h2o.sparkling.utils.ScalaUtils.withResource import ai.h2o.sparkling.utils.SparkSessionUtils import com.google.gson.{Gson, JsonElement} diff --git a/ml/src/main/scala/ai/h2o/sparkling/ml/algos/H2OGridSearch.scala b/ml/src/main/scala/ai/h2o/sparkling/ml/algos/H2OGridSearch.scala index 8f39ff6f3a..9032550969 100644 --- a/ml/src/main/scala/ai/h2o/sparkling/ml/algos/H2OGridSearch.scala +++ b/ml/src/main/scala/ai/h2o/sparkling/ml/algos/H2OGridSearch.scala @@ -18,13 +18,13 @@ package ai.h2o.sparkling.ml.algos import java.util +import ai.h2o.sparkling.H2OFrame import ai.h2o.sparkling.backend.exceptions.RestApiCommunicationException import ai.h2o.sparkling.backend.utils.{RestApiUtils, RestCommunication, RestEncodingUtils} -import ai.h2o.sparkling.frame.H2OFrame +import ai.h2o.sparkling.ml.internals.{H2OMetric, H2OModel, H2OModelCategory} import ai.h2o.sparkling.ml.models.{H2OMOJOModel, H2OMOJOSettings} import ai.h2o.sparkling.ml.params.H2OGridSearchParams import ai.h2o.sparkling.ml.utils.H2OParamsReadable -import ai.h2o.sparkling.model.{H2OMetric, H2OModel, H2OModelCategory} import ai.h2o.sparkling.utils.SparkSessionUtils import hex.Model import hex.grid.HyperSpaceSearchCriteria diff --git a/ml/src/main/scala/ai/h2o/sparkling/ml/algos/H2OKMeans.scala b/ml/src/main/scala/ai/h2o/sparkling/ml/algos/H2OKMeans.scala index a2da2516b8..5aea96e980 100644 --- a/ml/src/main/scala/ai/h2o/sparkling/ml/algos/H2OKMeans.scala +++ b/ml/src/main/scala/ai/h2o/sparkling/ml/algos/H2OKMeans.scala @@ -16,7 +16,7 @@ */ package ai.h2o.sparkling.ml.algos -import ai.h2o.sparkling.frame.{H2OColumnType, H2OFrame} +import ai.h2o.sparkling.{H2OColumnType, H2OFrame} import ai.h2o.sparkling.ml.params.H2OKMeansParams import hex.kmeans.KMeansModel.KMeansParameters import org.apache.spark.ml.util.{DefaultParamsReadable, Identifiable} diff --git a/ml/src/main/scala/ai/h2o/sparkling/ml/algos/H2OSupervisedAlgorithm.scala b/ml/src/main/scala/ai/h2o/sparkling/ml/algos/H2OSupervisedAlgorithm.scala index b70c74e5a2..0eab1dadbd 100644 --- a/ml/src/main/scala/ai/h2o/sparkling/ml/algos/H2OSupervisedAlgorithm.scala +++ b/ml/src/main/scala/ai/h2o/sparkling/ml/algos/H2OSupervisedAlgorithm.scala @@ -16,7 +16,7 @@ */ package ai.h2o.sparkling.ml.algos -import ai.h2o.sparkling.frame.{H2OColumnType, H2OFrame} +import ai.h2o.sparkling.{H2OColumnType, H2OFrame} import ai.h2o.sparkling.ml.models.H2OSupervisedMOJOModel import ai.h2o.sparkling.ml.params.H2OAlgoSupervisedParams import hex.Model diff --git a/ml/src/main/scala/ai/h2o/sparkling/ml/features/H2OTargetEncoder.scala b/ml/src/main/scala/ai/h2o/sparkling/ml/features/H2OTargetEncoder.scala index 5092487335..e82074475f 100644 --- a/ml/src/main/scala/ai/h2o/sparkling/ml/features/H2OTargetEncoder.scala +++ b/ml/src/main/scala/ai/h2o/sparkling/ml/features/H2OTargetEncoder.scala @@ -18,10 +18,10 @@ package ai.h2o.sparkling.ml.features import ai.h2o.sparkling.backend.utils.RestCommunication +import ai.h2o.sparkling.ml.internals.H2OModel import ai.h2o.sparkling.ml.models.{H2OTargetEncoderBase, H2OTargetEncoderModel} import ai.h2o.sparkling.ml.params.H2OAlgoParamsHelper import ai.h2o.sparkling.ml.utils.EstimatorCommonUtils -import ai.h2o.sparkling.model.H2OModel import ai.h2o.targetencoding._ import org.apache.spark.h2o.H2OContext import org.apache.spark.ml.Estimator diff --git a/ml/src/main/scala/ai/h2o/sparkling/ml/features/H2OTargetEncoderModelUtils.scala b/ml/src/main/scala/ai/h2o/sparkling/ml/features/H2OTargetEncoderModelUtils.scala index bc3ceab9bb..cc9c8095c7 100644 --- a/ml/src/main/scala/ai/h2o/sparkling/ml/features/H2OTargetEncoderModelUtils.scala +++ b/ml/src/main/scala/ai/h2o/sparkling/ml/features/H2OTargetEncoderModelUtils.scala @@ -17,7 +17,7 @@ package ai.h2o.sparkling.ml.features -import ai.h2o.sparkling.frame.H2OFrame +import ai.h2o.sparkling.H2OFrame import ai.h2o.sparkling.ml.params.H2OTargetEncoderParams trait H2OTargetEncoderModelUtils extends H2OTargetEncoderParams { diff --git a/core/src/main/scala/ai/h2o/sparkling/model/H2OMetric.java b/ml/src/main/scala/ai/h2o/sparkling/ml/internals/H2OMetric.java similarity index 97% rename from core/src/main/scala/ai/h2o/sparkling/model/H2OMetric.java rename to ml/src/main/scala/ai/h2o/sparkling/ml/internals/H2OMetric.java index 84f1386aea..cd16fb0f49 100644 --- a/core/src/main/scala/ai/h2o/sparkling/model/H2OMetric.java +++ b/ml/src/main/scala/ai/h2o/sparkling/ml/internals/H2OMetric.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package ai.h2o.sparkling.model; +package ai.h2o.sparkling.ml.internals; public enum H2OMetric { AUTO(true), diff --git a/core/src/main/scala/ai/h2o/sparkling/model/H2OMetricsHolder.scala b/ml/src/main/scala/ai/h2o/sparkling/ml/internals/H2OMetricsHolder.scala similarity index 71% rename from core/src/main/scala/ai/h2o/sparkling/model/H2OMetricsHolder.scala rename to ml/src/main/scala/ai/h2o/sparkling/ml/internals/H2OMetricsHolder.scala index cf41fb0814..dad2ef9bca 100644 --- a/core/src/main/scala/ai/h2o/sparkling/model/H2OMetricsHolder.scala +++ b/ml/src/main/scala/ai/h2o/sparkling/ml/internals/H2OMetricsHolder.scala @@ -15,8 +15,8 @@ * limitations under the License. */ -package ai.h2o.sparkling.model +package ai.h2o.sparkling.ml.internals -case class H2OMetricsHolder(trainingMetrics: Map[H2OMetric, Double], - validationMetrics: Map[H2OMetric, Double], - crossValidationMetrics: Map[H2OMetric, Double]) +private[sparkling] case class H2OMetricsHolder(trainingMetrics: Map[H2OMetric, Double], + validationMetrics: Map[H2OMetric, Double], + crossValidationMetrics: Map[H2OMetric, Double]) diff --git a/core/src/main/scala/ai/h2o/sparkling/model/H2OModel.scala b/ml/src/main/scala/ai/h2o/sparkling/ml/internals/H2OModel.scala similarity index 97% rename from core/src/main/scala/ai/h2o/sparkling/model/H2OModel.scala rename to ml/src/main/scala/ai/h2o/sparkling/ml/internals/H2OModel.scala index 5003a44539..9271e54a0d 100644 --- a/core/src/main/scala/ai/h2o/sparkling/model/H2OModel.scala +++ b/ml/src/main/scala/ai/h2o/sparkling/ml/internals/H2OModel.scala @@ -15,7 +15,7 @@ * limitations under the License. */ -package ai.h2o.sparkling.model +package ai.h2o.sparkling.ml.internals import java.io.File import java.nio.file.Files @@ -31,7 +31,7 @@ import water.api.schemas3.ModelsV3 import scala.collection.JavaConverters._ -class H2OModel private(val modelId: String, +private[sparkling] class H2OModel private(val modelId: String, val modelCategory: H2OModelCategory.Value, val metrics: H2OMetricsHolder, val trainingParams: Map[String, String]) @@ -57,7 +57,7 @@ class H2OModel private(val modelId: String, } } -object H2OModel extends RestCommunication { +private[sparkling] object H2OModel extends RestCommunication { private[sparkling] def listAllModels(): Array[String] = { val conf = H2OContext.ensure().getConf diff --git a/core/src/main/scala/ai/h2o/sparkling/model/H2OModelCategory.scala b/ml/src/main/scala/ai/h2o/sparkling/ml/internals/H2OModelCategory.scala similarity index 92% rename from core/src/main/scala/ai/h2o/sparkling/model/H2OModelCategory.scala rename to ml/src/main/scala/ai/h2o/sparkling/ml/internals/H2OModelCategory.scala index 0f07662dad..7484c5b32b 100644 --- a/core/src/main/scala/ai/h2o/sparkling/model/H2OModelCategory.scala +++ b/ml/src/main/scala/ai/h2o/sparkling/ml/internals/H2OModelCategory.scala @@ -15,12 +15,12 @@ * limitations under the License. */ -package ai.h2o.sparkling.model +package ai.h2o.sparkling.ml.internals /** * Copied from H2O's class ModelCategory */ -object H2OModelCategory extends Enumeration { +private[sparkling] object H2OModelCategory extends Enumeration { val Unknown, Binomial, Multinomial, Ordinal, Regression, HGLMRegression, Clustering, AutoEncoder, TargetEncoder, DimReduction, WordEmbedding, CoxPH, AnomalyDetection = Value diff --git a/ml/src/main/scala/ai/h2o/sparkling/ml/models/H2OTargetEncoderModel.scala b/ml/src/main/scala/ai/h2o/sparkling/ml/models/H2OTargetEncoderModel.scala index 937dfdcbdc..2cefa7f6cc 100644 --- a/ml/src/main/scala/ai/h2o/sparkling/ml/models/H2OTargetEncoderModel.scala +++ b/ml/src/main/scala/ai/h2o/sparkling/ml/models/H2OTargetEncoderModel.scala @@ -17,11 +17,11 @@ package ai.h2o.sparkling.ml.models +import ai.h2o.sparkling.H2OFrame import ai.h2o.sparkling.backend.utils.{RestApiUtils, RestCommunication} -import ai.h2o.sparkling.frame.H2OFrame +import ai.h2o.sparkling.ml.internals.H2OModel import ai.h2o.sparkling.ml.features.H2OTargetEncoderModelUtils import ai.h2o.sparkling.ml.utils.SchemaUtils -import ai.h2o.sparkling.model.H2OModel import org.apache.spark.h2o.H2OContext import org.apache.spark.ml.Model import org.apache.spark.ml.param.ParamMap diff --git a/ml/src/main/scala/ai/h2o/sparkling/ml/params/H2OGridSearchParams.scala b/ml/src/main/scala/ai/h2o/sparkling/ml/params/H2OGridSearchParams.scala index c17ceabf0e..9f7f5b4a95 100644 --- a/ml/src/main/scala/ai/h2o/sparkling/ml/params/H2OGridSearchParams.scala +++ b/ml/src/main/scala/ai/h2o/sparkling/ml/params/H2OGridSearchParams.scala @@ -19,7 +19,7 @@ package ai.h2o.sparkling.ml.params import java.util import ai.h2o.sparkling.ml.algos.{H2OGridSearch, H2OSupervisedAlgorithm} -import ai.h2o.sparkling.model.H2OMetric +import ai.h2o.sparkling.ml.internals.H2OMetric import hex.grid.HyperSpaceSearchCriteria import hex.{Model, ScoreKeeper} import org.apache.spark.ml.param._ diff --git a/ml/src/main/scala/ai/h2o/sparkling/ml/utils/EstimatorCommonUtils.scala b/ml/src/main/scala/ai/h2o/sparkling/ml/utils/EstimatorCommonUtils.scala index 3499f05ac3..6ef89fbf6d 100644 --- a/ml/src/main/scala/ai/h2o/sparkling/ml/utils/EstimatorCommonUtils.scala +++ b/ml/src/main/scala/ai/h2o/sparkling/ml/utils/EstimatorCommonUtils.scala @@ -16,8 +16,8 @@ */ package ai.h2o.sparkling.ml.utils +import ai.h2o.sparkling.backend.H2OJob import ai.h2o.sparkling.backend.utils.{RestApiUtils, RestCommunication} -import ai.h2o.sparkling.job.H2OJob import hex.schemas.ModelBuilderSchema import org.apache.spark.h2o.H2OContext diff --git a/ml/src/test/scala/ai/h2o/sparkling/ml/algos/H2OGLMTestSuite.scala b/ml/src/test/scala/ai/h2o/sparkling/ml/algos/H2OGLMTestSuite.scala index d850b13398..f8cc0ca351 100644 --- a/ml/src/test/scala/ai/h2o/sparkling/ml/algos/H2OGLMTestSuite.scala +++ b/ml/src/test/scala/ai/h2o/sparkling/ml/algos/H2OGLMTestSuite.scala @@ -17,7 +17,7 @@ package ai.h2o.sparkling.ml.algos -import ai.h2o.sparkling.model.H2OModel +import ai.h2o.sparkling.ml.internals.H2OModel import org.apache.spark.SparkContext import org.apache.spark.h2o.utils.SharedH2OTestContext import org.apache.spark.ml.{Pipeline, PipelineModel} diff --git a/py/src/ai/h2o/sparkling/ml/params/H2OGridSearchParams.py b/py/src/ai/h2o/sparkling/ml/params/H2OGridSearchParams.py index c5e8c1805f..8c55af3036 100644 --- a/py/src/ai/h2o/sparkling/ml/params/H2OGridSearchParams.py +++ b/py/src/ai/h2o/sparkling/ml/params/H2OGridSearchParams.py @@ -76,7 +76,7 @@ class H2OGridSearchParams(H2OCommonSupervisedParams): Params._dummy(), "selectBestModelBy", "Specifies the metric which is used for comparing and sorting the models returned by the grid.", - H2OTypeConverters.toEnumString("ai.h2o.sparkling.model.H2OMetric")) + H2OTypeConverters.toEnumString("ai.h2o.sparkling.ml.internals.H2OMetric")) parallelism = Param( Params._dummy(),