/
assemble
executable file
·136 lines (119 loc) · 4.65 KB
/
assemble
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
#!/bin/bash
source $STI_SCRIPTS_PATH/s2i-env-vars
function match_sum {
local sumfile=$1
local delim=$2
local md5=$3
local initial=$(cat $sumfile | cut -d"$delim" -f1 | tr [:upper:] [:lower:] | tr -d [:space:])
local rest=$(cat $sumfile | cut -d"$delim" --complement -f1 | tr [:upper:] [:lower:] | tr -d [:space:])
if [ "$md5" == "$initial" ] || [ "$md5" == "$rest" ]; then
return 0
fi
return 1
}
if [ -f $SPARK_HOME/bin/spark-submit ]; then
echo "Spark is installed"
else
echo "Attempting to install Spark"
# If a url has been specfified for spark use it
if [ -n "$SPARK_URL" ]; then
echo Downloading $SPARK_URL
wget $SPARK_URL -P $S2I_SOURCE_DIR
fi
if [ -n "$SPARK_MD5_URL" ]; then
echo Downloading $SPARK_MD5_URL
wget $SPARK_MD5_URL -P $S2I_SOURCE_DIR
fi
for spark in $(ls "$S2I_SOURCE_DIR"); do
spark=$S2I_SOURCE_DIR/$spark
# Is the file a directory? If it contains spark-submit, move it
if [ -d "$spark" ]; then
if ! [ -f $spark/bin/spark-submit ]; then
echo Ignoring directory $spark, no spark-submit
continue
fi
echo Installing from directory $spark
sparkdir=$SPARK_INSTALL/$(basename $spark)
mv $spark $SPARK_INSTALL
else
# If we can get the table of contents, it's a tar archive, otherwise ignore
tar -tf $spark &> /dev/null
if [ "$?" -ne 0 ]; then
echo Ignoring $spark, not a tar archive
continue
fi
# Does the tarball contain a spark-submit?
name=$(tar -tzf $spark | grep "spark-submit$")
if [ "$?" -ne 0 ]; then
echo Ignoring tarball $spark, no spark-submit
continue
else
# See if we have an md5 file to match against
if [ -f "$spark".md5 ]; then
calcvalue=$(md5sum "$spark" | cut -d\ -f1)
# split the md5 file using a space
match_sum "$spark".md5 \ $calcvalue
matched="$?"
if [ "$matched" -ne 0 ]; then
# split the md5 file using equals sign in case it's BSD
match_sum "$spark".md5 \= $calcvalue
matched="$?"
fi
if [ "$matched" -ne 0 ]; then
echo Ignoring tarball $spark, md5sum did not match
continue
fi
fi
# dname will be the intial directory from the path of spark-submit
# we found in the tarball, ie the dir created by tar
echo Installing from tarball $spark
dname=$(dirname $name | cut -d/ -f 1)
sparkdir=$SPARK_INSTALL/$dname
tar -xzf $spark -C $SPARK_INSTALL
fi
fi
ln -s $sparkdir $SPARK_INSTALL/distro
chmod -R g+rwX $SPARK_INSTALL/distro
# Search for the spark entrypoint file and copy it to $SPARK_INSTALL
entry=$(find $sparkdir/kubernetes -name entrypoint.sh)
if [ -n "$entry" ]; then
echo Copying spark entrypoint
cp $entry $SPARK_INSTALL
# We have to patch the entrypoint to toggle error checking
# around the uidentry check for 2.3 (fix on the way)
sed -i '/^uidentry/i set +e' $SPARK_INSTALL/entrypoint.sh
sed -i '/^uidentry/a set -e' $SPARK_INSTALL/entrypoint.sh
# We also want to get rid of the tini invocation
sed -i "s@exec /sbin/tini -s --@exec@" $SPARK_INSTALL/entrypoint.sh
fi
# Include the default spark configuration files
pushd $SPARK_INSTALL/conf
for conf_file in *; do
if ! [ -f "$SPARK_HOME"/conf/$conf_file ]; then
echo Moving $conf_file to $SPARK_HOME/conf
mv $conf_file $SPARK_HOME/conf
else
echo Not moving $conf_file, $SPARK_HOME/conf/$conf_file already exists
fi
done
popd
# Can we run spark-submit?
$SPARK_HOME/bin/spark-submit --version
if [ "$?" -eq 0 ]; then
echo Spark installed successfully
exit 0
else
echo Spark install failed
fi
# Just in case there is more than one tarball, clean up
rm -rf $sparkdir
done
echo no valid Spark distribution found
if [ -n "$DEBUG_ASSEMBLE" ]; then
echo Looping forever so you can \'oc rsh\'
while true; do
sleep 5
done
fi
exit 1
fi