Permalink
Switch branches/tags
Nothing to show
Find file Copy path
Fetching contributors…
Cannot retrieve contributors at this time
46 lines (38 sloc) 1.61 KB
################################################################################
# Host-specific Spark configuration
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
#
# Customize Peel configuration values appearing in
#
# https://github.com/stratosphere/peel/blob/master/peel-extensions/src/main/resources/reference.spark.conf
#
# here.
#
system {
spark {
config {
# spark-env.sh entries
env {
HADOOP_CONF_DIR = ${system.hadoop-2.path.config}
# enable this if you want to use spark with native libraries
# only use if there is a hadoop version compiled with native libraries for your environment!
# SPARK_DAEMON_JAVA_OPTS = "-Djava.library.path="${system.hadoop-2.path.home}"/lib/native"
# 28 GiB of memory
SPARK_EXECUTOR_MEMORY = "28672m"
SPARK_WORKER_MEMORY = "28672m"
}
# spark-defaults.conf
defaults {
spark.master = "spark://"${runtime.hostname}":7077"
# tmp folder for spilling data to disk (on node-local storage)
spark.local.dir = "/data/"${user.name}"/spark/tmp"
# 28 GiB of memory
spark.executor.memory = "28672m"
# memory of driver (e.g. to receive/gather results sets)
spark.driver.memory = "2048m"
# sets the memory limit of result sets gathered on driver to infinite
spark.driver.maxResultSize = "0"
}
}
}
}