forked from jey/alchemist
-
Notifications
You must be signed in to change notification settings - Fork 2
/
run-convert-2T-to-parquet.sh
33 lines (29 loc) · 1010 Bytes
/
run-convert-2T-to-parquet.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
#!/bin/bash
#SBATCH -q premium
#SBATCH -N 78
#SBATCH -t 02:00:00
#SBATCH -e mysparkjob_%j.err
#SBATCH -o mysparkjob_%j.out
#SBATCH -C haswell
#module load collectl
#start-collectl.sh
source setup/cori-start-alchemist.sh 39 2
sleep 15
infname=/global/cscratch1/sd/gittens/large-datasets/rda_ds093.0_dataset/outputs/ocean2T.h5
outfname=/global/cscratch1/sd/gittens/large-datasets/ocean2T.parquet
varname=rows
spark-submit --verbose\
--driver-memory 120G\
--executor-memory 120G\
--executor-cores 32 \
--driver-cores 32 \
--num-executors 38 \
--conf spark.driver.extraLibraryPath=$SCRATCH/alchemistSHELL/alchemist/lib\
--conf spark.executor.extraLibraryPath=$SCRATCH/alchemistSHELL/alchemist/lib\
--conf spark.eventLog.enabled=true\
--conf spark.eventLog.dir=$SCRATCH/spark/event_logs\
--class amplab.alchemist.ConvertH5ToParquet \
test/target/scala-2.11/alchemist-tests-assembly-0.0.2.jar $infname $outfname $varname | tee convert-h5-to-parquet.log
stop-all.sh
exit
#stop-collectl.sh