Merge pull request #675 from mv1388/pypi_package_install_for_prepare_…

…instance

Pypi package install for prepare instance

bin/AWS/create_instance.sh

@@ -21,10 +21,10 @@ function usage()
      -n, --name STR                 name for the created instance
      --no-bootstrap                 keep the newly created instance and don't run the bootstrapping
      -o, --os-name STR              username depending on the OS chosen. Default is ubuntu
-     -t, --terminate                the instance will be terminated when training is done
      -s, --ssh-start                automatically ssh into the instance when the training starts
      --on-demand                    create on-demand instance instead of spot instance
      --central-region               create the instance in the central region (Frankfurt)
+     --pypi                         install package from PyPI instead of the local package version
      -h, --help                     show this help message and exit
 
 HEREDOC
@@ -40,11 +40,11 @@ instance_config="default_config.json"
 instance_type=
 run_bootstrap=true
 username="ubuntu"
-terminate_cmd=false
 ssh_at_start=false
 spot_instance=true
 aws_region="eu-west-1"
 instance_name=
+local_pypi_install=""
 
 
 while [[ $# -gt 0 ]]; do
@@ -95,10 +95,6 @@ case $key in
     username="$2"
     shift 2 # past argument value
     ;;
-    -t|--terminate)
-    terminate_cmd=true
-    shift 1 # past argument value
-    ;;
     -s|--ssh-start)
     ssh_at_start=true
     shift 1 # past argument value
@@ -111,6 +107,10 @@ case $key in
     aws_region="eu-central-1"
     shift 1 # past argument value
     ;;
+    --pypi)
+    local_pypi_install="--pypi"
+    shift 1 # past argument value
+    ;;
     -h|--help )
     usage;
     exit;
@@ -137,13 +137,8 @@ else
     py_env="pytorch_latest_p36"
 fi
 
-terminate_setting=""
-if [ "$terminate_cmd" == true ]; then
-    terminate_setting="--terminate"
-fi
-
 if [[ "$instance_type" != "" ]]; then
-    instance_config=config_$(tr . _ <<< $instance_type).json
+    instance_type="--instance-type $instance_type"
 fi
 
 if [ "$aws_region" == "eu-central-1" ]; then
@@ -179,7 +174,7 @@ ec2_instance_address=$(aws ec2 describe-instances --instance-ids $instance_id --
 ##############################
 echo "Preparing instance"
 ./prepare_instance.sh -k $key_path -a $ec2_instance_address \
-    -f $DL_framework -v $AIToolbox_version -p $local_project_path -d $dataset_name -r $preproc_dataset -o $username --aws-region $aws_region --no-ssh
+    -f $DL_framework -v $AIToolbox_version -p $local_project_path -d $dataset_name -r $preproc_dataset -o $username --aws-region $aws_region $local_pypi_install --no-ssh
 
 
 #########################################################

bin/AWS/prepare_instance.sh

@@ -37,6 +37,7 @@ function usage()
      -r, --preproc STR      the preprocessed version of the main dataset
      -o, --os-name STR      username depending on the OS chosen. Default is ubuntu
      --aws-region STR       create the instance in the specified region. Default is Ireland (eu-west-1)
+     --pypi                 install package from PyPI instead of the local package version
      --no-ssh               disable auto ssh-ing to the instance
      -h, --help             show this help message and exit
 
@@ -52,6 +53,7 @@ dataset_name="None"
 preproc_dataset="None"
 username="ubuntu"
 aws_region="eu-west-1"
+pypi_install=false
 auto_ssh_to_instance=true
 
 while [[ $# -gt 0 ]]; do
@@ -94,6 +96,10 @@ case $key in
     aws_region="$2"
     shift 2 # past argument value
     ;;
+    --pypi)
+    pypi_install=true
+    shift 1 # past argument value
+    ;;
     --no-ssh)
     auto_ssh_to_instance=false
     shift 1 # past argument value
@@ -135,7 +141,9 @@ export AWS_DEFAULT_REGION=$aws_region
 
 ssh -i $key_path -o "StrictHostKeyChecking no" $username@$ec2_instance_address 'mkdir ~/project ; mkdir ~/project/data ; mkdir ~/project/model_results'
 
-scp -i $key_path ../../dist/aitoolbox-$AIToolbox_version.tar.gz  $username@$ec2_instance_address:~/project
+if [ $pypi_install == false ]; then
+    scp -i $key_path ../../dist/aitoolbox-$AIToolbox_version.tar.gz  $username@$ec2_instance_address:~/project
+fi
 scp -i $key_path download_data.sh  $username@$ec2_instance_address:~/project
 scp -i $key_path run_experiment.sh  $username@$ec2_instance_address:~/project
 
@@ -164,7 +172,11 @@ pip install seaborn==0.9.0
 #conda install -y -c conda-forge jsonnet
 #conda install -y -c anaconda seaborn=0.9.0
 
-pip install aitoolbox-$AIToolbox_version.tar.gz
+if [ $pypi_install == false ]; then
+  pip install aitoolbox-$AIToolbox_version.tar.gz
+else
+  pip install aitoolbox==$AIToolbox_version
+fi
 
 if [ $local_project_path != 'None' ]; then
     pip install -r ~/project/AWS_run_scripts/AWS_bootstrap/requirements.txt

bin/AWS/submit_job.sh

@@ -32,6 +32,7 @@ function usage()
      -s, --ssh-start                automatically ssh into the instance when the training starts
      --on-demand                    create on-demand instance instead of spot instance
      --central-region               create the instance in the central region (Frankfurt)
+     --pypi                         install package from PyPI instead of the local package version
      -h, --help                     show this help message and exit
 
 HEREDOC
@@ -53,6 +54,7 @@ terminate_cmd=false
 ssh_at_start=false
 spot_instance=true
 aws_region="eu-west-1"
+local_pypi_install=""
 instance_name=
 
 default_logging_filename="training.log"
@@ -134,6 +136,10 @@ case $key in
     aws_region="eu-central-1"
     shift 1 # past argument value
     ;;
+    --pypi)
+    local_pypi_install="--pypi"
+    shift 1 # past argument value
+    ;;
     -h|--help )
     usage;
     exit;
@@ -215,7 +221,7 @@ ec2_instance_address=$(aws ec2 describe-instances --instance-ids $instance_id --
 ##############################
 echo "Preparing instance"
 ./prepare_instance.sh -k $key_path -a $ec2_instance_address \
-    -f $DL_framework -v $AIToolbox_version -p $local_project_path -d $dataset_name -r $preproc_dataset -o $username --aws-region $aws_region --no-ssh
+    -f $DL_framework -v $AIToolbox_version -p $local_project_path -d $dataset_name -r $preproc_dataset -o $username --aws-region $aws_region $local_pypi_install --no-ssh
 
 
 #########################################################

requirements.txt

@@ -5,7 +5,7 @@ matplotlib
 seaborn
 torch>=1.6.0
 torchvision
-torchtext
+torchtext==0.7
 pytorch-nlp
 transformers
 joblib

setup.py

@@ -26,7 +26,7 @@
 
         'torch>=1.6.0',
         'torchvision',
-        'torchtext',
+        'torchtext==0.7',
         'pytorch-nlp',
         'transformers',