diff --git a/models/tf_details/resnet_run_loop.py b/models/tf_details/resnet_run_loop.py index 2fec782..6397cbd 100644 --- a/models/tf_details/resnet_run_loop.py +++ b/models/tf_details/resnet_run_loop.py @@ -327,7 +327,7 @@ def resnet_main(flags, model_function, input_function, opts = None): logging.warning("batch sizes differ in model %i %s", flags.batch_size, opts["batch_size"]) if ngpus > 1: - validate_batch_size_for_multi_gpu(bs) + validate_batch_size_for_multi_gpu(flags.batch_size) # There are two steps required if using multi-GPU: (1) wrap the model_fn, # and (2) wrap the optimizer. The first happens here, and (2) happens # in the model_fn itself when the optimizer is defined. diff --git a/scripts/tf-full/resnet50-singularity-bs128.slurm b/scripts/tf-full/resnet50-singularity-bs128.slurm index d7b674c..0f221b1 100644 --- a/scripts/tf-full/resnet50-singularity-bs128.slurm +++ b/scripts/tf-full/resnet50-singularity-bs128.slurm @@ -10,4 +10,8 @@ cd /home/steinba/development/deeprace/ pwd module load singularity/2.4.2 -singularity exec -B $PWD:/home/steinba/deeprace --nv /scratch/steinba/tf1.7-plus.simg python3 /home/steinba/development/deeprace/deeprace.py train -b tf -O batch_size=128 -c "k80:1,fs:nfs,singularity:lustre" -t /home/steinba/development/deeprace/scripts/tf-full/tf-full-resnet56v1-bs128-singularity.tsv resnet56v1 +TDIR=`mktemp -d` + +singularity exec -B $PWD:/home/steinba/deeprace --nv /scratch/steinba/tf1.7-plus.simg python3 /home/steinba/development/deeprace/deeprace.py train -R ${TDIR} -b tf -O batch_size=128 -c "k80:1,fs:nfs,singularity:lustre" -t /home/steinba/development/deeprace/scripts/tf-full/tf-full-resnet56v1-bs128-singularity.tsv resnet56v1 + +rm -rf ${TDIR} \ No newline at end of file diff --git a/scripts/tf-full/resnet50-singularity.slurm b/scripts/tf-full/resnet50-singularity.slurm index 606029f..00867fb 100644 --- a/scripts/tf-full/resnet50-singularity.slurm +++ b/scripts/tf-full/resnet50-singularity.slurm @@ -10,4 +10,8 @@ cd /home/steinba/development/deeprace/ pwd module load singularity/2.4.2 -singularity exec -B $PWD:/home/steinba/deeprace --nv /scratch/steinba/tf1.7-plus.simg python3 /home/steinba/development/deeprace/deeprace.py train -b tf -c "k80:1,fs:nfs,singularity:lustre" -t /home/steinba/development/deeprace/scripts/tf-full/tf-full-resnet56v1-singularity.tsv resnet56v1 +TDIR=`mktemp -d` + +singularity exec -B $PWD:/home/steinba/deeprace --nv /scratch/steinba/tf1.7-plus.simg python3 /home/steinba/development/deeprace/deeprace.py train -R ${TDIR} -b tf -c "k80:1,fs:nfs,singularity:lustre" -t /home/steinba/development/deeprace/scripts/tf-full/tf-full-resnet56v1-singularity.tsv resnet56v1 + +rm -rf ${TDIR} \ No newline at end of file