From afc1e1978b60a1589724771f8f9a5b94e3dd833c Mon Sep 17 00:00:00 2001 From: wangxiaoyu8 Date: Fri, 28 Aug 2015 09:05:28 +0800 Subject: [PATCH 1/2] SPARK-10314 [CORE] RDD persist to OFF_HEAP tachyon got block rdd_x_x not found exception when parallelism is big than data split size --- .../org/apache/spark/storage/BlockManagerMasterEndpoint.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/storage/BlockManagerMasterEndpoint.scala b/core/src/main/scala/org/apache/spark/storage/BlockManagerMasterEndpoint.scala index 7db6035553ae6..7375300289697 100644 --- a/core/src/main/scala/org/apache/spark/storage/BlockManagerMasterEndpoint.scala +++ b/core/src/main/scala/org/apache/spark/storage/BlockManagerMasterEndpoint.scala @@ -489,7 +489,7 @@ private[spark] class BlockManagerInfo( logInfo("Added %s on ExternalBlockStore on %s (size: %s)".format( blockId, blockManagerId.hostPort, Utils.bytesToString(externalBlockStoreSize))) } - if (!blockId.isBroadcast && blockStatus.isCached) { + if (!blockId.isBroadcast && (blockStatus.isCached || blockStatus.storageLevel.useOffHeap)) { _cachedBlocks += blockId } } else if (_blocks.containsKey(blockId)) { From 4cca4388237a87632cf942368ad8dab8d37a883b Mon Sep 17 00:00:00 2001 From: wangxiaoyu8 Date: Wed, 14 Oct 2015 15:37:25 +0800 Subject: [PATCH 2/2] SPARK-11100 HiveThriftServer not registering with Zookeeper --- .../sql/hive/thriftserver/HiveThriftServer2.scala | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2.scala index a4fd0c3ce9702..e4ab168379f49 100644 --- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2.scala +++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2.scala @@ -57,6 +57,11 @@ object HiveThriftServer2 extends Logging { val server = new HiveThriftServer2(sqlContext) server.init(sqlContext.hiveconf) server.start() + if(SparkSQLEnv.hiveContext.hiveconf.getBoolVar( + ConfVars.HIVE_SERVER2_SUPPORT_DYNAMIC_SERVICE_DISCOVERY)) { + invoke(classOf[HiveServer2], server, "addServerInstanceToZooKeeper", + classOf[HiveConf] -> SparkSQLEnv.hiveContext.hiveconf) + } listener = new HiveThriftServer2Listener(server, sqlContext.conf) sqlContext.sparkContext.addSparkListener(listener) uiTab = if (sqlContext.sparkContext.getConf.getBoolean("spark.ui.enabled", true)) { @@ -84,6 +89,11 @@ object HiveThriftServer2 extends Logging { val server = new HiveThriftServer2(SparkSQLEnv.hiveContext) server.init(SparkSQLEnv.hiveContext.hiveconf) server.start() + if(SparkSQLEnv.hiveContext.hiveconf.getBoolVar( + ConfVars.HIVE_SERVER2_SUPPORT_DYNAMIC_SERVICE_DISCOVERY)) { + invoke(classOf[HiveServer2], server, "addServerInstanceToZooKeeper", + classOf[HiveConf] -> SparkSQLEnv.hiveContext.hiveconf) + } logInfo("HiveThriftServer2 started") listener = new HiveThriftServer2Listener(server, SparkSQLEnv.hiveContext.conf) SparkSQLEnv.sparkContext.addSparkListener(listener)