spark_defaults_autoscale.conf

#spark.jars                                          /home/cdsw/dist/lib/bigdl-0.2.0-SNAPSHOT-jar-with-dependencies.jar
# assume 4 nodes 3 exec per node   16cpu 64gig ram
#  executors   4 nodes x 3 exe per node = 12 exec
# exec cores 4 cores x 3 exe per node = 12 of 16 cpus per node
# memory at 18 gig per executor x 3 = 54gig of 64 gig ram per node 
spark.executor.memory                               18g
spark.executor.cores                                4
spark.executor.instances                            12
#spark.dynamicAllocation.maxExecutors            10
spark.scheduler.listenerbus.eventqueue.size  10000