From 85285d890ce1c741ca39abe7c070cf3d30f87bdd Mon Sep 17 00:00:00 2001 From: pengyanhong Date: Tue, 4 Nov 2014 23:03:39 +0800 Subject: [PATCH 1/2] fix bug SPARK-3033:java.math.BigDecimal cannot be cast to org.apache.hadoop.hive.common.type.HiveDecimal --- .../scala/org/apache/spark/sql/hive/HiveInspectors.scala | 1 + .../main/scala/org/apache/spark/sql/hive/hiveUdfs.scala | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala index 58815daa82276..78aff9d30549f 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala @@ -184,6 +184,7 @@ private[hive] trait HiveInspectors { case l: Byte => l: java.lang.Byte case b: BigDecimal => HiveShim.createDecimal(b.underlying()) case d: Decimal => HiveShim.createDecimal(d.toBigDecimal.underlying()) + case d: org.apache.hadoop.hive.common.`type`.HiveDecimal => d case b: Array[Byte] => b case d: java.sql.Date => d case t: java.sql.Timestamp => t diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUdfs.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUdfs.scala index 86f7eea5dfd69..e5126c9c5cf08 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUdfs.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUdfs.scala @@ -38,6 +38,7 @@ import org.apache.spark.util.Utils.getContextOrSparkClassLoader /* Implicit conversions */ import scala.collection.JavaConversions._ +import org.apache.hadoop.hive.common.`type`.HiveDecimal private[hive] abstract class HiveFunctionRegistry extends analysis.FunctionRegistry with HiveInspectors { @@ -181,6 +182,12 @@ private[hive] case class HiveGenericUdf(functionClassName: String, children: Seq () => { children(idx).eval(input) }) + if (deferedObjects(i).get().isInstanceOf[java.math.BigDecimal] == true) { + val decimal = deferedObjects(i).get().asInstanceOf[java.math.BigDecimal] + deferedObjects(i).asInstanceOf[DeferredObjectAdapter].set(() => { + HiveDecimal.create(decimal).asInstanceOf[EvaluatedType] + }) + } i += 1 } unwrap(function.evaluate(deferedObjects), returnInspector) From 36e2b9ec2b6c0929f8259c75f1f5c45127857ad4 Mon Sep 17 00:00:00 2001 From: pengyanhong Date: Thu, 6 Nov 2014 22:27:37 +0800 Subject: [PATCH 2/2] SPARK-2815: Compilation failed upon the hadoop version 2.0.0-cdh4.5.0 --- .../apache/spark/deploy/yarn/YarnAllocationHandler.scala | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala index abd37834ed3cc..310e2010156bb 100644 --- a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala +++ b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala @@ -19,18 +19,16 @@ package org.apache.spark.deploy.yarn import java.util.concurrent.CopyOnWriteArrayList import java.util.concurrent.atomic.AtomicInteger - import scala.collection.JavaConversions._ import scala.collection.mutable.{ArrayBuffer, HashMap} - import org.apache.spark.{SecurityManager, SparkConf} import org.apache.spark.scheduler.SplitInfo - import org.apache.hadoop.conf.Configuration import org.apache.hadoop.yarn.api.AMRMProtocol import org.apache.hadoop.yarn.api.records._ import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest import org.apache.hadoop.yarn.util.Records +import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse /** * Acquires resources for executors from a ResourceManager and launches executors in new containers. @@ -118,7 +116,7 @@ private[yarn] class YarnAllocationHandler( request.getPriority, request.getCapability)) } - new AlphaAllocateResponse(resourceManager.allocate(req).getAMResponse()) + new AlphaAllocateResponse(resourceManager.allocate(req)) } override protected def releaseContainer(container: Container) = { @@ -220,7 +218,7 @@ private[yarn] class YarnAllocationHandler( retval } - private class AlphaAllocateResponse(response: AMResponse) extends YarnAllocateResponse { + private class AlphaAllocateResponse(response: AllocateResponse) extends YarnAllocateResponse { override def getAllocatedContainers() = response.getAllocatedContainers() override def getAvailableResources() = response.getAvailableResources() override def getCompletedContainersStatuses() = response.getCompletedContainersStatuses()