From a30ca9f4cfde295a811cbe144d6cf165be1227c2 Mon Sep 17 00:00:00 2001 From: Sylvain Zimmer Date: Tue, 26 Jul 2016 17:47:26 -0400 Subject: [PATCH 1/2] [SPARK-16740][SQL] Fix Long overflow in LongToUnsafeRowMap --- .../org/apache/spark/sql/execution/joins/HashedRelation.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala index 412e8c54ca308..a9e4832ef2cf6 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala @@ -608,7 +608,7 @@ private[execution] final class LongToUnsafeRowMap(val mm: TaskMemoryManager, cap def optimize(): Unit = { val range = maxKey - minKey // Convert to dense mode if it does not require more memory or could fit within L1 cache - if (range < array.length || range < 1024) { + if (range >= 0 && (range < array.length || range < 1024)) { try { ensureAcquireMemory((range + 1) * 8L) } catch { From f35b7f663c0e8c68ae62712cf8bef5c3769934a8 Mon Sep 17 00:00:00 2001 From: Sylvain Zimmer Date: Tue, 26 Jul 2016 18:27:51 -0400 Subject: [PATCH 2/2] Add a comment about [SPARK-16740], as suggested by @srowen --- .../org/apache/spark/sql/execution/joins/HashedRelation.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala index a9e4832ef2cf6..cf4454c033384 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala @@ -608,6 +608,7 @@ private[execution] final class LongToUnsafeRowMap(val mm: TaskMemoryManager, cap def optimize(): Unit = { val range = maxKey - minKey // Convert to dense mode if it does not require more memory or could fit within L1 cache + // SPARK-16740: Make sure range doesn't overflow if minKey has a large negative value if (range >= 0 && (range < array.length || range < 1024)) { try { ensureAcquireMemory((range + 1) * 8L)