Skip to content

Commit

Permalink
Add logs.
Browse files Browse the repository at this point in the history
  • Loading branch information
yhuai committed Jun 25, 2015
1 parent 7bac2fe commit b22cf7d
Showing 1 changed file with 7 additions and 0 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,7 @@ object SparkSubmitClassLoaderTest extends Logging {
val sc = new SparkContext(conf)
val hiveContext = new TestHiveContext(sc)
val df = hiveContext.createDataFrame((1 to 100).map(i => (i, i))).toDF("i", "j")
logInfo("Testing load classes at the driver side.")
// First, we load classes at driver side.
try {
Class.forName(args(0), true, Thread.currentThread().getContextClassLoader)
Expand All @@ -124,6 +125,7 @@ object SparkSubmitClassLoaderTest extends Logging {
throw new Exception("Could not load user class from jar:\n", t)
}
// Second, we load classes at the executor side.
logInfo("Testing load classes at the executor side.")
val result = df.mapPartitions { x =>
var exception: String = null
try {
Expand All @@ -141,6 +143,7 @@ object SparkSubmitClassLoaderTest extends Logging {
}

// Load a Hive UDF from the jar.
logInfo("Registering temporary Hive UDF provided in a jar.")
hiveContext.sql(
"""
|CREATE TEMPORARY FUNCTION example_max
Expand All @@ -150,18 +153,22 @@ object SparkSubmitClassLoaderTest extends Logging {
hiveContext.createDataFrame((1 to 10).map(i => (i, s"str$i"))).toDF("key", "val")
source.registerTempTable("sourceTable")
// Load a Hive SerDe from the jar.
logInfo("Creating a Hive table with a SerDe provided in a jar.")
hiveContext.sql(
"""
|CREATE TABLE t1(key int, val string)
|ROW FORMAT SERDE 'org.apache.hive.hcatalog.data.JsonSerDe'
""".stripMargin)
// Actually use the loaded UDF and SerDe.
logInfo("Writing data into the table.")
hiveContext.sql(
"INSERT INTO TABLE t1 SELECT example_max(key) as key, val FROM sourceTable GROUP BY val")
logInfo("Running a simple query on the table.")
val count = hiveContext.table("t1").orderBy("key", "val").count()
if (count != 10) {
throw new Exception(s"table t1 should have 10 rows instead of $count rows")
}
logInfo("Test finishes.")
}
}

Expand Down

0 comments on commit b22cf7d

Please sign in to comment.