Skip to content

Commit

Permalink
Move parsing and escaping logic to utils.sh
Browse files Browse the repository at this point in the history
This commit also fixes a deadly typo.
  • Loading branch information
andrewor14 committed Aug 7, 2014
1 parent dec2343 commit a4df3c4
Show file tree
Hide file tree
Showing 3 changed files with 82 additions and 59 deletions.
43 changes: 10 additions & 33 deletions bin/spark-class
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,9 @@ FWDIR="$(cd `dirname $0`/..; pwd)"
# Export this as SPARK_HOME
export SPARK_HOME="$FWDIR"

# Load utility functions
. "$SPARK_HOME/bin/utils.sh"

. $FWDIR/bin/load-spark-env.sh

if [ -z "$1" ]; then
Expand Down Expand Up @@ -77,7 +80,7 @@ case "$1" in
'org.apache.spark.deploy.SparkSubmit')
OUR_JAVA_OPTS="$SPARK_JAVA_OPTS $SPARK_SUBMIT_OPTS"
if [ -n "$SPARK_SUBMIT_LIBRARY_PATH" ]; then
OUR_JAVA_OPTS="$OUT_JAVA_OPTS -Djava.library.path=$SPARK_SUBMIT_LIBRARY_PATH"
OUR_JAVA_OPTS="$OUR_JAVA_OPTS -Djava.library.path=$SPARK_SUBMIT_LIBRARY_PATH"
fi
OUR_JAVA_MEM=${SPARK_DRIVER_MEMORY:-$DEFAULT_MEM}
;;
Expand All @@ -103,11 +106,16 @@ fi
# Set JAVA_OPTS to be able to load native libraries and to set heap size
JAVA_OPTS="-XX:MaxPermSize=128m $OUR_JAVA_OPTS"
JAVA_OPTS="$JAVA_OPTS -Xms$OUR_JAVA_MEM -Xmx$OUR_JAVA_MEM"

# Load extra JAVA_OPTS from conf/java-opts, if it exists
if [ -e "$FWDIR/conf/java-opts" ] ; then
JAVA_OPTS="$JAVA_OPTS `cat $FWDIR/conf/java-opts`"
fi
export JAVA_OPTS

# Escape JAVA_OPTS properly to handle whitespace, double quotes and backslashes
# This exports the escaped java options into ESCAPED_JAVA_OPTS
escape_java_options "$JAVA_OPTS"

# Attention: when changing the way the JAVA_OPTS are assembled, the change must be reflected in CommandUtils.scala!

TOOLS_DIR="$FWDIR"/tools
Expand Down Expand Up @@ -148,37 +156,6 @@ if $cygwin; then
fi
export CLASSPATH

# Properly escape java options, dealing with whitespace, double quotes and backslashes
# This accepts a string, and returns the escaped list through ESCAPED_JAVA_OPTS
escape_java_options() {
ESCAPED_JAVA_OPTS=() # return value
option_buffer="" # buffer for collecting parts of an option
opened_quotes=0 # whether we are expecting a closing double quotes
for word in $1; do
contains_quote=$(echo "$word" | grep \" | grep -v \\\\\")
if [ -n "$contains_quote" ]; then
# Flip the bit
opened_quotes=$(((opened_quotes + 1) % 2))
fi
if [[ $opened_quotes == 0 ]]; then
ESCAPED_JAVA_OPTS+=("$(echo "$option_buffer $word" | sed "s/^[[:space:]]*//" | sed "s/\([^\\]\)\"/\1/g")")
option_buffer=""
else
option_buffer="$option_buffer $word"
fi
done
# Something is wrong if we ended with open double quotes
if [[ $opened_quotes == 1 ]]; then
echo "Java options parse error! Expecting closing double quotes." 1>&2
exit 1
fi
}

escape_java_options "$JAVA_OPTS"
for option in "${ESCAPED_JAVA_OPTS[@]}"; do
echo "$option"
done

if [ "$SPARK_PRINT_LAUNCH_COMMAND" == "1" ]; then
echo -n "Spark Command: " 1>&2
echo "$RUNNER" -cp "$CLASSPATH" "${ESCAPED_JAVA_OPTS[@]}" "$@" 1>&2
Expand Down
39 changes: 13 additions & 26 deletions bin/spark-submit
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,14 @@
export SPARK_HOME="$(cd `dirname $0`/..; pwd)"
ORIG_ARGS=("$@")

# Load utility functions
. "$SPARK_HOME/bin/utils.sh"

# For client mode, the driver will be launched in the JVM that launches
# SparkSubmit, so we need to handle the class paths, java options, and
# memory pre-emptively in bash. Otherwise, it will be too late by the
# time the JVM has started.

while (($#)); do
if [ "$1" = "--deploy-mode" ]; then
DEPLOY_MODE=$2
Expand All @@ -40,34 +48,14 @@ done
DEPLOY_MODE=${DEPLOY_MODE:-"client"}
PROPERTIES_FILE=${PROPERTIES_FILE:-"$SPARK_HOME/conf/spark-defaults.conf"}

# For client mode, the driver will be launched in the JVM that launches
# SparkSubmit, so we need to handle the class paths, java options, and
# memory pre-emptively in bash. Otherwise, it will be too late by the
# time the JVM has started.

if [ $DEPLOY_MODE == "client" ]; then
# We parse the default properties file here, assuming each line is
# a key value pair delimited either by white space or "=" sign. All
# spark.driver.* configs must be processed now before it's too late.
# Parse the default properties file here for spark.driver.* configs
if [ -f "$PROPERTIES_FILE" ]; then
echo "Using properties file $PROPERTIES_FILE." 1>&2

# Parse the value of the given config according to the specifications outlined in
# http://docs.oracle.com/javase/7/docs/api/java/util/Properties.html#load(java.io.Reader)
parse_config() {
result=$( \
sed "/^[#!]/ d" "conf/spark-defaults.conf" | \
grep "$1" | \
sed "s/$1//" | \
sed "s/^[[:space:]]*[:=]\{0,1\}//" | \
sed "s/^[[:space:]]*\(.*\)[[:space:]]*$/\1/g" \
)
}
parse_config "spark.driver.memory"; DRIVER_MEMORY_CONF="$result"
parse_config "spark.driver.extraJavaOptions"; DRIVER_EXTRA_JAVA_OPTS="$result"
parse_config "spark.driver.extraClassPath"; DRIVER_EXTRA_CLASSPATH="$result"
parse_config "spark.driver.extraLibraryPath"; DRIVER_EXTRA_LIBRARY_PATH="$result"

parse_java_property "spark.driver.memory"; DRIVER_MEMORY_CONF="$JAVA_PROPERTY_VALUE"
parse_java_property "spark.driver.extraJavaOptions"; DRIVER_EXTRA_JAVA_OPTS="$JAVA_PROPERTY_VALUE"
parse_java_property "spark.driver.extraClassPath"; DRIVER_EXTRA_CLASSPATH="$JAVA_PROPERTY_VALUE"
parse_java_property "spark.driver.extraLibraryPath"; DRIVER_EXTRA_LIBRARY_PATH="$JAVA_PROPERTY_VALUE"
if [ -n "$DRIVER_EXTRA_JAVA_OPTS" ]; then
export SPARK_SUBMIT_OPTS="$SPARK_SUBMIT_OPTS $DRIVER_EXTRA_JAVA_OPTS"
fi
Expand All @@ -80,7 +68,6 @@ if [ $DEPLOY_MODE == "client" ]; then
else
echo "Warning: properties file $PROPERTIES_FILE does not exist!" 1>&2
fi

# Favor command line memory over config memory
DRIVER_MEMORY=${DRIVER_MEMORY:-"$DRIVER_MEMORY_CONF"}
if [ -n "$DRIVER_MEMORY" ]; then
Expand Down
59 changes: 59 additions & 0 deletions bin/utils.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

# Parse the value of a config from a java properties file according to the specifications in
# http://docs.oracle.com/javase/7/docs/api/java/util/Properties.html#load(java.io.Reader).
# This accepts the name of the config and returns the value through JAVA_PROPERTY_VALUE.
parse_java_property() {
JAVA_PROPERTY_VALUE=$( \
sed "/^[#!]/ d" "conf/spark-defaults.conf" | \
grep "$1" | \
sed "s/$1//" | \
sed "s/^[[:space:]]*[:=]\{0,1\}//" | \
sed "s/^[[:space:]]*\(.*\)[[:space:]]*$/\1/g" \
)
export JAVA_PROPERTY_VALUE
}

# Properly escape java options, dealing with whitespace, double quotes and backslashes
# This accepts a string, and returns the escaped list through ESCAPED_JAVA_OPTS.
escape_java_options() {
ESCAPED_JAVA_OPTS=() # return value
option_buffer="" # buffer for collecting parts of an option
opened_quotes=0 # whether we are expecting a closing double quotes
for word in $1; do
contains_quote=$(echo "$word" | grep \" | grep -v \\\\\")
if [ -n "$contains_quote" ]; then
# Flip the bit
opened_quotes=$(((opened_quotes + 1) % 2))
fi
if [[ $opened_quotes == 0 ]]; then
ESCAPED_JAVA_OPTS+=("$(echo "$option_buffer $word" | sed "s/^[[:space:]]*//" | sed "s/\([^\\]\)\"/\1/g")")
option_buffer=""
else
option_buffer="$option_buffer $word"
fi
done
# Something is wrong if we ended with open double quotes
if [[ $opened_quotes == 1 ]]; then
echo "Java options parse error! Expecting closing double quotes." 1>&2
exit 1
fi
export ESCAPED_JAVA_OPTS
}

0 comments on commit a4df3c4

Please sign in to comment.