/
jenkins-test
executable file
·372 lines (295 loc) · 10.2 KB
/
jenkins-test
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
#!/usr/bin/env bash
set -e -x -v
# navigate to mango parent directory, set in Jenkins
if [ -z "$WORKSPACE" ]
then
echo "WORKSPACE variable is not set. Defaulting to current directory.."
WORKSPACE="$(echo $PWD)"
fi
# make a tempdir for writing maven cruft to
MANGO_MVN_TMP_DIR=$(mktemp -d -t mangoTestMvnXXXXXXX)
# make a temp directory for installing/downloading jars to
MANGO_TMP_DIR=$(mktemp -d -t mangoTestXXXXXXX)
MANGO_TMP_DIR=$MANGO_TMP_DIR/deleteMePleaseThisIsNoLongerNeeded
mkdir $MANGO_TMP_DIR
# run integration tests
# set the TMPDIR envar, which is used by python to choose where to make temp directories
export TMPDIR=${MANGO_TMP_DIR}
# Copy workspace into our temp space for testing
cp -rT $WORKSPACE $MANGO_TMP_DIR
# new project root is tmp dir, where everything was copied
PROJECT_ROOT=$MANGO_TMP_DIR
pushd $PROJECT_ROOT
# add this tempdir to the poms...
find . -name pom.xml \
-exec sed -i.bak \
-e "s:sun.io.serialization.extendedDebugInfo=true:sun.io.serialization.extendedDebugInfo=true -Djava.io.tmpdir=${MANGO_MVN_TMP_DIR}:g" \
{} \;
find . -name "*.bak" -exec rm {} \;
# variable declarations
export PATH=${JAVA_HOME}/bin/:${PATH}
export MAVEN_OPTS="-Xmx1536m -XX:MaxPermSize=1g -Dfile.encoding=utf-8"
VERSION=$(grep "<version>" ${PROJECT_ROOT}/pom.xml | head -2 | tail -1 | sed 's/ *<version>//g' | sed 's/<\/version>//g')
# unique identifier for conda env
PYTHON_UUID=$(uuidgen)
# function that deletes conda environments and temporary environments
function clean_up {
set +x +v
# deactivate and remove all mango associated conda envs
source deactivate
# delete conda env, if it exists
conda remove -y --force --name mango-build-${PYTHON_UUID} --all || \
echo "conda ENV mango-build-${PYTHON_UUID} does not exist"
# delete tmp directories
rm -rf ${MANGO_TMP_DIR}
rm -rf ${MANGO_MVN_TMP_DIR}
set -x -v
}
trap clean_up EXIT
# is the hadoop version set?
if ! [[ ${HADOOP_VERSION} ]];
then
echo "HADOOP_VERSION environment variable is not set."
echo "Please set this variable before running."
exit 1
fi
# is the spark version set?
if ! [[ ${SPARK_VERSION} ]];
then
echo "SPARK_VERSION environment variable is not set."
echo "Please set this variable before running."
exit 1
fi
set -e
# move to Scala 2.11 if requested
if [ ${SCALAVER} == 2.11 ];
then
set +e
./scripts/move_to_scala_2.11.sh
set -e
fi
# move to Scala 2.12 if requested
if [ ${SCALAVER} == 2.12 ];
then
set +e
./scripts/move_to_scala_2.12.sh
set -e
fi
# move to Spark 2.x if requested
if [ ${SPARK_VERSION} == 2.4.6 ];
then
set +e
./scripts/move_to_spark_2.sh
set -e
fi
# print versions
echo "Testing MANGO version ${VERSION} on Spark ${SPARK_VERSION} and Hadoop ${HADOOP_VERSION}"
# clean the targets first
mvn clean
# clear distribution in case there are residual jars
rm -rf mango-distribution/target/
# if this is a pull request, we need to set the coveralls pr id
if [[ ! -z $ghprbPullId ]];
then
COVERALLS_PRB_OPTION="-DpullRequest=${ghprbPullId}"
fi
# coveralls token should not be visible
set +x +v
if [[ -z ${COVERALLS_REPO_TOKEN} ]];
then
echo "Coveralls token is not set. Exiting..."
exit 1
fi
# if those pass, run tests
mvn -U \
test \
-P core,coverage,coveralls scoverage:report coveralls:report \
-DrepoToken=${COVERALLS_REPO_TOKEN} ${COVERALLS_PRB_OPTION}
# make verbose again
set -x -v
# if those pass, build the distribution package
mvn -U \
-P distribution \
package \
-DskipTests \
-Dhadoop.version=${HADOOP_VERSION} \
-Dspark.version=${SPARK_VERSION} \
-DargLine=${MANGO_MVN_TMP_DIR}
# make sure that the distribution package contains an assembly jar
# if no assembly jar is found, this will exit with code 1 and fail the build
tar tzf mango-distribution/target/mango-distribution*-bin.tar.gz | \
grep mango-assembly | \
grep jar | \
grep -v -e sources -e javadoc
# we are done with maven, so clean up the maven temp dir
find ${MANGO_MVN_TMP_DIR}
rm -rf ${MANGO_MVN_TMP_DIR}
find . -name pom.xml \
-exec sed -i.bak \
-e "s:sun.io.serialization.extendedDebugInfo=true -Djava.io.tmpdir=${MANGO_MVN_TMP_DIR}:sun.io.serialization.extendedDebugInfo=true:g" \
{} \;
find . -name "*.bak" -exec rm -f {} \;
# set spark artifact string for downloading assembly
SPARK=spark-${SPARK_VERSION}
# what hadoop version are we on? format string for downloading spark assembly
if [[ $HADOOP_VERSION =~ ^2\.7 ]]; then
HADOOP=hadoop2.7
else
echo "Unknown Hadoop version."
exit 1
fi
# Spark 2.4.3+ and less than 3.0.0 needs special case for Scala 2.12
if [ ${SCALAVER} == 2.12 ] && [ ${SPARK_VERSION} == 2.4.6 ]
then
curl \
-L "https://www.apache.org/dyn/mirrors/mirrors.cgi?action=download&filename=spark/${SPARK}/${SPARK}-bin-without-hadoop-scala-2.12.tgz" \
-o ${MANGO_TMP_DIR}/${SPARK}-bin-without-hadoop-scala-2.12.tgz
tar xzf ${MANGO_TMP_DIR}/${SPARK}-bin-without-hadoop-scala-2.12.tgz --directory ${MANGO_TMP_DIR}
export SPARK_HOME=${MANGO_TMP_DIR}/${SPARK}-bin-without-hadoop-scala-2.12
curl \
-L "https://www.apache.org/dyn/mirrors/mirrors.cgi?action=download&filename=hadoop/core/hadoop-2.7.7/hadoop-2.7.7.tar.gz" \
-o ${MANGO_TMP_DIR}/hadoop-2.7.7.tar.gz
tar xzf ${MANGO_TMP_DIR}/hadoop-2.7.7.tar.gz --directory ${MANGO_TMP_DIR}
# remove references to avro 1.7.x
find ${MANGO_TMP_DIR}/hadoop-2.7.7 -name *.jar | grep avro | xargs rm
export SPARK_DIST_CLASSPATH=$(${MANGO_TMP_DIR}/hadoop-2.7.7/bin/hadoop classpath)
else
curl \
-v \
-L "https://www.apache.org/dyn/mirrors/mirrors.cgi?action=download&filename=spark/${SPARK}/${SPARK}-bin-${HADOOP}.tgz" \
-o ${MANGO_TMP_DIR}/${SPARK}-bin-${HADOOP}.tgz
tar xzf ${MANGO_TMP_DIR}/${SPARK}-bin-${HADOOP}.tgz --directory ${MANGO_TMP_DIR}
export SPARK_HOME=${MANGO_TMP_DIR}/${SPARK}-bin-${HADOOP}
fi
# create a conda environment for python build, if necessary
pythons=( 3.6 )
for python in ${pythons[*]}
do
set +x +v
conda create -y -q -c conda-forge -n mango-build-${PYTHON_UUID} python=${python} pip=19.3.1 jupyterlab=2.0.1
source activate mango-build-${PYTHON_UUID}
set -x -v
# install npm and node in the venv for mango-pileup
conda install -y nodejs
python -h
pip install cigar==0.1.3
python -h
pip install matplotlib==2.0.2
python -h
pip install ipython==7.8.0
python -h
pip install jupyter==1.0.0
python -h
pip install bdgenomics.adam==0.30.0
python -h
pip install bdgenomics.mango.pileup==0.0.6a0
python -h
pip install ipywidgets==7.0.0
python -h
pip install psutil
python -h
pip install pytest==3.9.1
python -h
pip install jupyter_contrib_nbextensions
python -h
# prepare mango python
pushd mango-python
make prepare
popd
python -h
# we can run the python build, now that we have a spark executable
# requires SPARK_HOME to be set
mvn -U \
-P python,distribution \
package \
-DskipTests \
-Dhadoop.version=${HADOOP_VERSION} \
-Dspark.version=${SPARK_VERSION}
# clean mango python
pushd mango-python
make clean
popd
# prepare mango viz
pushd mango-pileup
make prepare
popd
# run the widgets tests (mango-pileup)
mvn -P widgets test
# clean mango viz
pushd mango-pileup
make clean
popd
# make sure that the distribution package contains an egg
# if no assembly jar is found, this will exit with code 1 and fail the build
tar tzf mango-distribution/target/mango-distribution*-bin.tar.gz | \
grep bdgenomics.mango | \
grep egg
# test mango notebook, output to temporary log file
MANGO_NOTEBOOK_LOGS=${MANGO_TMP_DIR}/mango-notebook.log
# notebook args --ip=0.0.0.0 --no-browser --allow-root required for docker
bash -xv ${PROJECT_ROOT}/bin/mango-notebook -- --ip=0.0.0.0 --no-browser --allow-root &> ${MANGO_NOTEBOOK_LOGS} &
# sleep for 2 seconds to let mango-notebook start up
sleep 2
# if mango-notebook fails, then trying to kill the process will fail
if kill $! ; then
echo "Mango Notebook Succeeded"
else
echo "Mango Notebook failed with logs:"
cat ${MANGO_NOTEBOOK_LOGS}
exit 1
fi
# deactivate and remove the conda env
set +e +x
source deactivate
conda remove -y --name mango-build-${PYTHON_UUID} --all
set -e -x
done
# test mango submit
MANGO_BROWSER_LOGS=${MANGO_TMP_DIR}/mango-submit.log
bash -xv ${PROJECT_ROOT}/example-files/browser-scripts/run-example.sh &> ${MANGO_BROWSER_LOGS} &
# sleep for 10 seconds to let mango-browser start up
sleep 10
# make sure server responds to POST for reads and variants
curl -s -N \
--header "Content-type: application/json" \
--request POST \
--data "{\"pageToken\":null,\"pageSize\":200,\"readGroupIds\":[\"chr17_7500000-7515000_sam\"],\"referenceId\":\"chr17\",\"start\":1,\"end\":100}" \
http://localhost:8080/reads/search | grep alignments
if [[ $? != 0 ]]; then
echo
exit 1
fi
curl -s -N \
--header "Content-type: application/json" \
--request POST \
--data "{\"variantSetId\":\"ALL_chr17_7500000-7515000_phase3_shapeit2_mvncall_integrated_v5a_20130502_genotypes_vcf\",\"pageToken\":null,\"pageSize\":400,\"referenceName\":\"chr17\",\"callSetIds\":[],\"start\":7500000,\"end\":7515000}" \
http://localhost:8080/variants/search | grep variants
if [[ $? != 0 ]]; then
echo "invalid variants"
exit 1
fi
# get processes with mango prefix
PROCESSES=( $(ps aux | grep -F 'browser-scripts/run-example.sh' | grep -v -F 'grep' | awk '{ print $2 }') )
# There should be processes running for mango-browser
if [ "${#PROCESSES[@]}" -gt 0 ] ; then
echo "Mango Browser Succeeded"
# kill all Mango Browser related processes
kill ${PROCESSES};
else
echo "Mango Browser failed with logs:"
cat ${MANGO_BROWSER_LOGS}
exit 1
fi
# test genome builds
TEST_GENOME="anoGam1"
bash -xv ${PROJECT_ROOT}/bin/make_genome ${TEST_GENOME} ${MANGO_TMP_DIR}
unzip ${MANGO_TMP_DIR}/${TEST_GENOME}.genome -d ${MANGO_TMP_DIR}/${TEST_GENOME}
pushd ${MANGO_TMP_DIR}/${TEST_GENOME}
cat properties.txt
cat ${TEST_GENOME}.chrom.sizes
popd
# exist mango working directory
popd
echo
echo "All the tests passed"
echo