Skip to content

Commit

Permalink
[KYUUBI apache#6211] Check memory offHeap enabled for CustomResourceP…
Browse files Browse the repository at this point in the history
…rofileExec

# 🔍 Description
## Issue References 🔗

This pull request fixes #

## Describe Your Solution 🔧

We should check `spark.memory.offHeap.enabled` when applying for `executorOffHeapMemory`.

## Types of changes 🔖

- [X] Bugfix (non-breaking change which fixes an issue)
- [ ] New feature (non-breaking change which adds functionality)
- [ ] Breaking change (fix or feature that would cause existing functionality to change)

## Test Plan 🧪

#### Behavior Without This Pull Request ⚰️

#### Behavior With This Pull Request 🎉

#### Related Unit Tests

---

# Checklist 📝

- [X] This patch was not authored or co-authored using [Generative Tooling](https://www.apache.org/legal/generative-tooling.html)

**Be nice. Be informative.**

Closes apache#6211 from wForget/hotfix.

Closes apache#6211

1c7c8cd [wforget] Check memory offHeap enabled for CustomResourceProfileExec

Authored-by: wforget <643348094@qq.com>
Signed-off-by: wforget <643348094@qq.com>
  • Loading branch information
wForget committed Mar 28, 2024
1 parent ae9e88f commit 9114e50
Show file tree
Hide file tree
Showing 3 changed files with 18 additions and 3 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,12 @@ case class CustomResourceProfileExec(child: SparkPlan) extends UnaryExecNode {
private val executorMemoryOverhead =
conf.getConf(FINAL_WRITE_STAGE_EXECUTOR_MEMORY_OVERHEAD)
.getOrElse(sparkContext.getConf.get("spark.executor.memoryOverhead", "1G"))
private val executorOffHeapMemory = conf.getConf(FINAL_WRITE_STAGE_EXECUTOR_OFF_HEAP_MEMORY)
private val executorOffHeapMemory =
if (sparkContext.getConf.getBoolean("spark.memory.offHeap.enabled", false)) {
conf.getConf(FINAL_WRITE_STAGE_EXECUTOR_OFF_HEAP_MEMORY)
} else {
None
}

override lazy val metrics: Map[String, SQLMetric] = {
val base = Map(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,12 @@ case class CustomResourceProfileExec(child: SparkPlan) extends UnaryExecNode {
private val executorMemoryOverhead =
conf.getConf(FINAL_WRITE_STAGE_EXECUTOR_MEMORY_OVERHEAD)
.getOrElse(sparkContext.getConf.get("spark.executor.memoryOverhead", "1G"))
private val executorOffHeapMemory = conf.getConf(FINAL_WRITE_STAGE_EXECUTOR_OFF_HEAP_MEMORY)
private val executorOffHeapMemory =
if (sparkContext.getConf.getBoolean("spark.memory.offHeap.enabled", false)) {
conf.getConf(FINAL_WRITE_STAGE_EXECUTOR_OFF_HEAP_MEMORY)
} else {
None
}

override lazy val metrics: Map[String, SQLMetric] = {
val base = Map(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,12 @@ case class CustomResourceProfileExec(child: SparkPlan) extends UnaryExecNode {
private val executorMemoryOverhead =
conf.getConf(FINAL_WRITE_STAGE_EXECUTOR_MEMORY_OVERHEAD)
.getOrElse(sparkContext.getConf.get("spark.executor.memoryOverhead", "1G"))
private val executorOffHeapMemory = conf.getConf(FINAL_WRITE_STAGE_EXECUTOR_OFF_HEAP_MEMORY)
private val executorOffHeapMemory =
if (sparkContext.getConf.getBoolean("spark.memory.offHeap.enabled", false)) {
conf.getConf(FINAL_WRITE_STAGE_EXECUTOR_OFF_HEAP_MEMORY)
} else {
None
}

override lazy val metrics: Map[String, SQLMetric] = {
val base = Map(
Expand Down

0 comments on commit 9114e50

Please sign in to comment.