Skip to content

Commit

Permalink
[SW-2089] Expose only H2OFrame, hide remaining internal API (#1967)
Browse files Browse the repository at this point in the history
  • Loading branch information
jakubhava committed Mar 19, 2020
1 parent 4ad8fdc commit 075ead8
Show file tree
Hide file tree
Showing 35 changed files with 53 additions and 57 deletions.
Expand Up @@ -15,7 +15,7 @@
* limitations under the License.
*/

package ai.h2o.sparkling.frame
package ai.h2o.sparkling

case class H2OColumn(
name: String,
Expand Down
Expand Up @@ -15,7 +15,7 @@
* limitations under the License.
*/

package ai.h2o.sparkling.frame
package ai.h2o.sparkling

object H2OColumnType extends Enumeration {
val enum, string, int, real, time, uuid = Value
Expand Down
Expand Up @@ -15,16 +15,15 @@
* limitations under the License.
*/

package ai.h2o.sparkling.frame
package ai.h2o.sparkling

import java.text.MessageFormat

import ai.h2o.sparkling.backend.NodeDesc
import ai.h2o.sparkling.backend.utils.RestApiUtils._
import ai.h2o.sparkling.backend.utils.{RestCommunication, RestEncodingUtils}
import ai.h2o.sparkling.backend.{H2OChunk, H2OJob, NodeDesc}
import ai.h2o.sparkling.extensions.rest.api.Paths
import ai.h2o.sparkling.extensions.rest.api.schema.{FinalizeFrameV3, InitializeFrameV3}
import ai.h2o.sparkling.job.H2OJob
import ai.h2o.sparkling.utils.Base64Encoding
import org.apache.spark.h2o.{H2OConf, H2OContext}
import water.api.schemas3.FrameChunksV3.FrameChunkV3
Expand All @@ -34,7 +33,7 @@ import water.api.schemas3._
/**
* H2OFrame representation via Rest API
*/
class H2OFrame private(val frameId: String, val columns: Array[H2OColumn], val chunks: Array[H2OChunk])
class H2OFrame private(val frameId: String, val columns: Array[H2OColumn], private[sparkling] val chunks: Array[H2OChunk])
extends Serializable with RestEncodingUtils {
private val conf = H2OContext.ensure("H2OContext needs to be running in order to create H2OFrame").getConf
val columnNames: Array[String] = columns.map(_.name)
Expand Down
Expand Up @@ -15,20 +15,19 @@
* limitations under the License.
*/

package ai.h2o.sparkling.frame
package ai.h2o.sparkling.backend

import java.io.{InputStream, OutputStream}

import ai.h2o.sparkling.backend.NodeDesc
import ai.h2o.sparkling.backend.utils.{RestApiUtils, RestCommunication}
import ai.h2o.sparkling.extensions.rest.api.Paths
import ai.h2o.sparkling.utils.{Base64Encoding, Compression}
import org.apache.spark.h2o.H2OConf


case class H2OChunk(index: Int, numberOfRows: Int, location: NodeDesc)
private[sparkling] case class H2OChunk(index: Int, numberOfRows: Int, location: NodeDesc)

object H2OChunk extends RestCommunication {
private[sparkling] object H2OChunk extends RestCommunication {
def getChunkAsInputStream(
node: NodeDesc,
conf: H2OConf,
Expand Down
Expand Up @@ -17,10 +17,9 @@

package ai.h2o.sparkling.backend

import ai.h2o.sparkling.SparkTimeZone
import ai.h2o.sparkling.backend.utils.SupportedTypes._
import ai.h2o.sparkling.backend.utils.{ConversionUtils, ReflectionUtils}
import ai.h2o.sparkling.frame.H2OFrame
import ai.h2o.sparkling.{H2OFrame, SparkTimeZone}
import org.apache.spark.h2o.H2OContext
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.types.DataType
Expand Down
Expand Up @@ -18,7 +18,7 @@
package ai.h2o.sparkling.backend

import ai.h2o.sparkling.backend.utils.ReflectionUtils
import ai.h2o.sparkling.frame.{H2OColumn, H2OColumnType, H2OFrame}
import ai.h2o.sparkling.{H2OColumn, H2OColumnType, H2OFrame}
import org.apache.spark.h2o.H2OContext
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.sources.{BaseRelation, PrunedScan, TableScan}
Expand Down
Expand Up @@ -15,15 +15,15 @@
* limitations under the License.
*/

package ai.h2o.sparkling.job
package ai.h2o.sparkling.backend

import ai.h2o.sparkling.backend.utils.RestApiUtils.getClusterEndpoint
import ai.h2o.sparkling.backend.utils.RestCommunication
import org.apache.spark.expose.Logging
import org.apache.spark.h2o.{H2OConf, H2OContext}
import water.api.schemas3.{JobV3, JobsV3}

class H2OJob private(val id: String) extends Logging {
private[sparkling] class H2OJob private(val id: String) extends Logging {
private val conf = H2OContext.ensure("H2OContext needs to be running!").getConf

private def fetchStatus(): H2OJobStatus.Value = {
Expand All @@ -48,7 +48,7 @@ class H2OJob private(val id: String) extends Logging {
}
}

object H2OJob extends RestCommunication {
private[sparkling] object H2OJob extends RestCommunication {
def apply(jobId: String): H2OJob = {
val conf = H2OContext.ensure().getConf
verifyAndGetJob(conf, jobId)
Expand Down
Expand Up @@ -15,7 +15,7 @@
* limitations under the License.
*/

package ai.h2o.sparkling.job
package ai.h2o.sparkling.backend

object H2OJobStatus extends Enumeration {
val DONE, CANCELLED, FAILED, RUNNING = Value
Expand Down
3 changes: 1 addition & 2 deletions core/src/main/scala/ai/h2o/sparkling/backend/H2ORDD.scala
Expand Up @@ -19,9 +19,8 @@ package ai.h2o.sparkling.backend

import java.lang.reflect.Constructor

import ai.h2o.sparkling.SparkTimeZone
import ai.h2o.sparkling.backend.utils.{ConversionUtils, ProductType}
import ai.h2o.sparkling.frame.H2OFrame
import ai.h2o.sparkling.{H2OFrame, SparkTimeZone}
import org.apache.spark.h2o.H2OContext
import org.apache.spark.{Partition, TaskContext}

Expand Down
Expand Up @@ -17,7 +17,7 @@

package ai.h2o.sparkling.backend

import ai.h2o.sparkling.frame.H2OFrame
import ai.h2o.sparkling.H2OFrame
import org.apache.spark.Partition

/**
Expand Down
1 change: 0 additions & 1 deletion core/src/main/scala/ai/h2o/sparkling/backend/Reader.scala
Expand Up @@ -23,7 +23,6 @@ import ai.h2o.sparkling.backend.converters.TimeZoneConversions
import ai.h2o.sparkling.backend.utils.SupportedTypes
import ai.h2o.sparkling.backend.utils.SupportedTypes._
import ai.h2o.sparkling.extensions.serde.ChunkAutoBufferReader
import ai.h2o.sparkling.frame.H2OChunk
import org.apache.spark.h2o.H2OConf
import org.apache.spark.unsafe.types.UTF8String

Expand Down
2 changes: 1 addition & 1 deletion core/src/main/scala/ai/h2o/sparkling/backend/Writer.scala
Expand Up @@ -20,9 +20,9 @@ package ai.h2o.sparkling.backend
import java.io.Closeable
import java.util.TimeZone

import ai.h2o.sparkling.H2OFrame
import ai.h2o.sparkling.backend.converters.TimeZoneConverter
import ai.h2o.sparkling.extensions.serde.{ChunkAutoBufferWriter, SerdeUtils}
import ai.h2o.sparkling.frame.{H2OChunk, H2OFrame}
import ai.h2o.sparkling.utils.ScalaUtils.withResource
import ai.h2o.sparkling.utils.SparkSessionUtils
import org.apache.spark.h2o.{H2OContext, RDD}
Expand Down
Expand Up @@ -47,7 +47,7 @@ object SparkDataFrameConverter extends Logging {

def toDataFrame[T <: Frame](hc: H2OContext, fr: T, copyMetadata: Boolean): DataFrame = {
DKV.put(fr)
toDataFrame(hc, ai.h2o.sparkling.frame.H2OFrame(fr._key.toString), copyMetadata)
toDataFrame(hc, ai.h2o.sparkling.H2OFrame(fr._key.toString), copyMetadata)
}

/**
Expand All @@ -59,7 +59,7 @@ object SparkDataFrameConverter extends Logging {
* @return a new DataFrame definition using given H2OFrame as data source
*/

def toDataFrame(hc: H2OContext, fr: ai.h2o.sparkling.frame.H2OFrame, copyMetadata: Boolean): DataFrame = {
def toDataFrame(hc: H2OContext, fr: ai.h2o.sparkling.H2OFrame, copyMetadata: Boolean): DataFrame = {
val spark = SparkSessionUtils.active
val relation = H2OFrameRelation(fr, copyMetadata)(spark.sqlContext)
spark.baseRelationToDataFrame(relation)
Expand Down
Expand Up @@ -43,11 +43,11 @@ object SupportedRDDConverter {
/** Transform H2OFrame to RDD */
def toRDD[A <: Product : TypeTag : ClassTag, T <: Frame](hc: H2OContext, fr: T): RDD[A] = {
DKV.put(fr)
toRDD(hc, ai.h2o.sparkling.frame.H2OFrame(fr._key.toString))
toRDD(hc, ai.h2o.sparkling.H2OFrame(fr._key.toString))
}

/** Transform H2OFrame to RDD */
def toRDD[A <: Product : TypeTag : ClassTag](hc: H2OContext, fr: ai.h2o.sparkling.frame.H2OFrame): RDD[A] = {
def toRDD[A <: Product : TypeTag : ClassTag](hc: H2OContext, fr: ai.h2o.sparkling.H2OFrame): RDD[A] = {
new H2ORDD[A](fr)(hc)
}
}
Expand Up @@ -18,7 +18,7 @@
package ai.h2o.sparkling.backend.utils

import ai.h2o.sparkling.backend.utils.SupportedTypes._
import ai.h2o.sparkling.frame.{H2OColumn, H2OColumnType}
import ai.h2o.sparkling.{H2OColumn, H2OColumnType}
import org.apache.spark.sql.types._
import water.api.API
import water.fvec.Vec
Expand Down
Expand Up @@ -62,7 +62,7 @@ class DefaultSource extends RelationProvider
schema: StructType): H2OFrameRelation = {
val key = checkKey(parameters)

H2OFrameRelation(ai.h2o.sparkling.frame.H2OFrame(key), copyMetadata = true)(sqlContext)
H2OFrameRelation(ai.h2o.sparkling.H2OFrame(key), copyMetadata = true)(sqlContext)
}

override def createRelation(sqlContext: SQLContext,
Expand Down
4 changes: 2 additions & 2 deletions core/src/main/scala/org/apache/spark/h2o/H2OContext.scala
Expand Up @@ -235,7 +235,7 @@ class H2OContext private(private val conf: H2OConf) extends H2OContextExtensions
def apply[T <: Frame](fr: T): RDD[A] = SupportedRDDConverter.toRDD[A, T](H2OContext.this, fr)
}

def asRDD[A <: Product : TypeTag : ClassTag](fr: ai.h2o.sparkling.frame.H2OFrame): org.apache.spark.rdd.RDD[A] = {
def asRDD[A <: Product : TypeTag : ClassTag](fr: ai.h2o.sparkling.H2OFrame): org.apache.spark.rdd.RDD[A] = {
SupportedRDDConverter.toRDD[A](this, fr)
}

Expand All @@ -246,7 +246,7 @@ class H2OContext private(private val conf: H2OConf) extends H2OContextExtensions
}

def asDataFrame(s: String, copyMetadata: Boolean): DataFrame = {
val frame = ai.h2o.sparkling.frame.H2OFrame(s)
val frame = ai.h2o.sparkling.H2OFrame(s)
SparkDataFrameConverter.toDataFrame(this, frame, copyMetadata)
}

Expand Down
Expand Up @@ -16,6 +16,7 @@
*/
package ai.h2o.sparkling.frame

import ai.h2o.sparkling.{H2OColumnType, H2OFrame}
import org.apache.spark.SparkContext
import org.apache.spark.h2o.utils.SharedH2OTestContext
import org.junit.runner.RunWith
Expand Down
Expand Up @@ -16,7 +16,7 @@
*/
package ai.h2o.sparkling.ml.algos

import ai.h2o.sparkling.frame.H2OFrame
import ai.h2o.sparkling.H2OFrame
import ai.h2o.sparkling.ml.params.H2OCommonParams
import ai.h2o.sparkling.ml.utils.{EstimatorCommonUtils, SchemaUtils}
import org.apache.spark.h2o.H2OContext
Expand Down
Expand Up @@ -16,12 +16,12 @@
*/
package ai.h2o.sparkling.ml.algos

import ai.h2o.sparkling.H2OFrame
import ai.h2o.sparkling.backend.exceptions.RestApiCommunicationException
import ai.h2o.sparkling.backend.utils.RestCommunication
import ai.h2o.sparkling.frame.H2OFrame
import ai.h2o.sparkling.ml.internals.H2OModel
import ai.h2o.sparkling.ml.models.{H2OMOJOModel, H2OMOJOSettings}
import ai.h2o.sparkling.ml.params.H2OAlgoCommonParams
import ai.h2o.sparkling.model.H2OModel
import hex.Model
import org.apache.spark.annotation.DeveloperApi
import org.apache.spark.ml.Estimator
Expand Down
4 changes: 2 additions & 2 deletions ml/src/main/scala/ai/h2o/sparkling/ml/algos/H2OAutoML.scala
Expand Up @@ -16,12 +16,12 @@
*/
package ai.h2o.sparkling.ml.algos

import ai.h2o.sparkling.H2OFrame
import ai.h2o.sparkling.backend.utils.{RestApiUtils, RestCommunication}
import ai.h2o.sparkling.frame.H2OFrame
import ai.h2o.sparkling.ml.internals.H2OModel
import ai.h2o.sparkling.ml.models.{H2OMOJOModel, H2OMOJOSettings}
import ai.h2o.sparkling.ml.params._
import ai.h2o.sparkling.ml.utils.H2OParamsReadable
import ai.h2o.sparkling.model.H2OModel
import ai.h2o.sparkling.utils.ScalaUtils.withResource
import ai.h2o.sparkling.utils.SparkSessionUtils
import com.google.gson.{Gson, JsonElement}
Expand Down
Expand Up @@ -18,13 +18,13 @@ package ai.h2o.sparkling.ml.algos

import java.util

import ai.h2o.sparkling.H2OFrame
import ai.h2o.sparkling.backend.exceptions.RestApiCommunicationException
import ai.h2o.sparkling.backend.utils.{RestApiUtils, RestCommunication, RestEncodingUtils}
import ai.h2o.sparkling.frame.H2OFrame
import ai.h2o.sparkling.ml.internals.{H2OMetric, H2OModel, H2OModelCategory}
import ai.h2o.sparkling.ml.models.{H2OMOJOModel, H2OMOJOSettings}
import ai.h2o.sparkling.ml.params.H2OGridSearchParams
import ai.h2o.sparkling.ml.utils.H2OParamsReadable
import ai.h2o.sparkling.model.{H2OMetric, H2OModel, H2OModelCategory}
import ai.h2o.sparkling.utils.SparkSessionUtils
import hex.Model
import hex.grid.HyperSpaceSearchCriteria
Expand Down
Expand Up @@ -16,7 +16,7 @@
*/
package ai.h2o.sparkling.ml.algos

import ai.h2o.sparkling.frame.{H2OColumnType, H2OFrame}
import ai.h2o.sparkling.{H2OColumnType, H2OFrame}
import ai.h2o.sparkling.ml.params.H2OKMeansParams
import hex.kmeans.KMeansModel.KMeansParameters
import org.apache.spark.ml.util.{DefaultParamsReadable, Identifiable}
Expand Down
Expand Up @@ -16,7 +16,7 @@
*/
package ai.h2o.sparkling.ml.algos

import ai.h2o.sparkling.frame.{H2OColumnType, H2OFrame}
import ai.h2o.sparkling.{H2OColumnType, H2OFrame}
import ai.h2o.sparkling.ml.models.H2OSupervisedMOJOModel
import ai.h2o.sparkling.ml.params.H2OAlgoSupervisedParams
import hex.Model
Expand Down
Expand Up @@ -18,10 +18,10 @@
package ai.h2o.sparkling.ml.features

import ai.h2o.sparkling.backend.utils.RestCommunication
import ai.h2o.sparkling.ml.internals.H2OModel
import ai.h2o.sparkling.ml.models.{H2OTargetEncoderBase, H2OTargetEncoderModel}
import ai.h2o.sparkling.ml.params.H2OAlgoParamsHelper
import ai.h2o.sparkling.ml.utils.EstimatorCommonUtils
import ai.h2o.sparkling.model.H2OModel
import ai.h2o.targetencoding._
import org.apache.spark.h2o.H2OContext
import org.apache.spark.ml.Estimator
Expand Down
Expand Up @@ -17,7 +17,7 @@

package ai.h2o.sparkling.ml.features

import ai.h2o.sparkling.frame.H2OFrame
import ai.h2o.sparkling.H2OFrame
import ai.h2o.sparkling.ml.params.H2OTargetEncoderParams

trait H2OTargetEncoderModelUtils extends H2OTargetEncoderParams {
Expand Down
Expand Up @@ -15,7 +15,7 @@
* limitations under the License.
*/

package ai.h2o.sparkling.model;
package ai.h2o.sparkling.ml.internals;

public enum H2OMetric {
AUTO(true),
Expand Down
Expand Up @@ -15,8 +15,8 @@
* limitations under the License.
*/

package ai.h2o.sparkling.model
package ai.h2o.sparkling.ml.internals

case class H2OMetricsHolder(trainingMetrics: Map[H2OMetric, Double],
validationMetrics: Map[H2OMetric, Double],
crossValidationMetrics: Map[H2OMetric, Double])
private[sparkling] case class H2OMetricsHolder(trainingMetrics: Map[H2OMetric, Double],
validationMetrics: Map[H2OMetric, Double],
crossValidationMetrics: Map[H2OMetric, Double])
Expand Up @@ -15,7 +15,7 @@
* limitations under the License.
*/

package ai.h2o.sparkling.model
package ai.h2o.sparkling.ml.internals

import java.io.File
import java.nio.file.Files
Expand All @@ -31,7 +31,7 @@ import water.api.schemas3.ModelsV3
import scala.collection.JavaConverters._


class H2OModel private(val modelId: String,
private[sparkling] class H2OModel private(val modelId: String,
val modelCategory: H2OModelCategory.Value,
val metrics: H2OMetricsHolder,
val trainingParams: Map[String, String])
Expand All @@ -57,7 +57,7 @@ class H2OModel private(val modelId: String,
}
}

object H2OModel extends RestCommunication {
private[sparkling] object H2OModel extends RestCommunication {

private[sparkling] def listAllModels(): Array[String] = {
val conf = H2OContext.ensure().getConf
Expand Down
Expand Up @@ -15,12 +15,12 @@
* limitations under the License.
*/

package ai.h2o.sparkling.model
package ai.h2o.sparkling.ml.internals

/**
* Copied from H2O's class ModelCategory
*/
object H2OModelCategory extends Enumeration {
private[sparkling] object H2OModelCategory extends Enumeration {
val Unknown, Binomial, Multinomial, Ordinal, Regression, HGLMRegression,
Clustering, AutoEncoder, TargetEncoder, DimReduction, WordEmbedding, CoxPH,
AnomalyDetection = Value
Expand Down

0 comments on commit 075ead8

Please sign in to comment.