Merge pull request #722 from mv1388/break_complex_bash_commands_into_…

…multiline_temp_files

Split long bash commands into multiple lines for better readability

bin/AWS/submit_job.sh

@@ -237,11 +237,24 @@ printf "\n========================================================\n"
 echo "Running the job"
 if [ "$run_with_scheduler" == true ]; then
   ssh -i $key_path $username@$ec2_instance_address \
-      "source activate $py_env ; tmux new-session -d -s 'training' 'source finish_prepare_instance.sh ; cd project ; python ~/training_job_scheduler.py add-job --experiment-script $experiment_script_file ; python ~/training_job_scheduler.py run $terminate_setting $log_upload_setting --aws-region $aws_region' \; pipe-pane 'cat > $logging_path'"
+      "source activate $py_env ;\
+      tmux new-session -d -s 'training' \
+      'source finish_prepare_instance.sh ;\
+        cd project ;\
+        python ~/training_job_scheduler.py add-job --experiment-script $experiment_script_file ;\
+        python ~/training_job_scheduler.py run $terminate_setting $log_upload_setting --aws-region $aws_region' \
+      \; \
+      pipe-pane 'cat > $logging_path'"
 
 else
   ssh -i $key_path $username@$ec2_instance_address \
-      "source activate $py_env ; tmux new-session -d -s 'training' 'source finish_prepare_instance.sh ; cd project ; ./run_experiment.sh $terminate_setting --experiment-script $experiment_script_file $log_upload_setting --cleanup-script --aws-region $aws_region' \; pipe-pane 'cat > $logging_path'"
+      "source activate $py_env ;\
+      tmux new-session -d -s 'training' \
+      'source finish_prepare_instance.sh ;\
+        cd project ;\
+        ./run_experiment.sh $terminate_setting --experiment-script $experiment_script_file $log_upload_setting --cleanup-script --aws-region $aws_region' \
+      \; \
+      pipe-pane 'cat > $logging_path'"
 fi
 
 echo "Instance IP: $ec2_instance_address"

bin/AWS/test_core_pytorch_compare.sh

@@ -172,7 +172,18 @@ fi
 #########################################################
 echo "Running the comparison tests"
 ssh -i $key_path $username@$ec2_instance_address \
-    "source activate $py_env ; tmux new-session -d -s 'training' 'export AWS_DEFAULT_REGION=$aws_region ; cd package_test ; pip install pytest datasets ; pip install -r requirements.txt ; python -m pytest $pytest_dir -s ; aws s3 cp $logging_path s3://aitoolbox-testing/core_pytorch_comparisson_testing/$logging_filename ; ~/send_log_email.sh -f $email_address -t $email_address -s \"GPU testing results for: $pytest_dir\" --attachment-path $logging_path ; $terminate_setting' \; pipe-pane 'cat > $logging_path'"
+    "source activate $py_env ;\
+    tmux new-session -d -s 'training' \
+    'export AWS_DEFAULT_REGION=$aws_region ;\
+      cd package_test ;\
+      pip install pytest datasets ;\
+      pip install -r requirements.txt ;\
+      python -m pytest $pytest_dir -s ;\
+      aws s3 cp $logging_path s3://aitoolbox-testing/core_pytorch_comparison_testing/$logging_filename ;\
+      ~/send_log_email.sh -f $email_address -t $email_address -s \"GPU testing results for: $pytest_dir\" --attachment-path $logging_path ;\
+      $terminate_setting' \
+    \; \
+    pipe-pane 'cat > $logging_path'"
 
 echo "Instance IP: $ec2_instance_address"
 

bin/AWS/update_experiments_on_AWS.sh

@@ -9,14 +9,14 @@ function usage()
 
    arguments:
      -a, --address STR        ec2 instance Public DNS address
-     -p, --project STR        path to the project to be optionally uploaded to the running ec2 instance
-     -h, --help               show this help message and exit
+     -p, --project STR        path to the project to be uploaded to the running ec2 instance
 
    optional arguments:
      -k, --key STR            path to ssh key
      --add-job                add training job to the running training job scheduler
      --experiment-script STR  name of the experiment bash script to be executed in order to start the training
      --aws-project-root STR   path to the aws-based project root
+     -h, --help               show this help message and exit
 
 HEREDOC
 }
@@ -27,7 +27,7 @@ local_project_path=
 
 add_training_scheduler_job=false
 experiment_script_file="aws_run_experiments_project.sh"
-aws_project_root_path=~/project
+aws_project_root_path="~/project"
 
 username="ubuntu"
 py_env="pytorch"
@@ -85,5 +85,7 @@ echo Re-ploading project folder $local_project_path
 source $local_project_path/AWS_run_scripts/AWS_core_scripts/aws_project_upload.sh $key_path $ec2_instance_address $aws_project_root_path $local_project_path
 
 if [ $add_training_scheduler_job == true ]; then
-    ssh -i $key_path $username@$ec2_instance_address "source activate $py_env ; python ~/training_job_scheduler.py add-job --experiment-script $experiment_script_file --project-root $aws_project_root_path"
+    ssh -i $key_path $username@$ec2_instance_address \
+        "source activate $py_env ;\
+          python ~/training_job_scheduler.py add-job --experiment-script $experiment_script_file --project-root $aws_project_root_path"
 fi

bin/AWS/update_package_on_AWS.sh

@@ -9,13 +9,13 @@ function usage()
 
    arguments:
      -a, --address STR      ec2 instance Public DNS address
-     -f, --framework STR    desired deep learning framework
-     -v, --version FLOAT    AIToolbox version to be installed on ec2
-     -h, --help             show this help message and exit
 
    optional arguments:
+     -f, --framework STR    desired deep learning framework
+     -v, --version FLOAT    AIToolbox version to be installed on ec2
      -k, --key STR          path to ssh key
      -o, --os-name STR      username depending on the OS chosen. Default is ubuntu
+     -h, --help             show this help message and exit
 
 HEREDOC
 }
@@ -80,4 +80,7 @@ fi
 
 scp -i $key_path ../../dist/aitoolbox-$AIToolbox_version.tar.gz  $username@$ec2_instance_address:~/project
 
-ssh -i $key_path $username@$ec2_instance_address "source activate $py_env ; pip uninstall aitoolbox ; pip install ~/project/aitoolbox-$AIToolbox_version.tar.gz"
+ssh -i $key_path $username@$ec2_instance_address \
+    "source activate $py_env ;\
+      pip uninstall aitoolbox ;\
+      pip install ~/project/aitoolbox-$AIToolbox_version.tar.gz"