Merge pull request #713 from mv1388/gpu-test-send-email

Send email report for GPU tests
mv1388 · Jul 31, 2022 · a48333e · a48333e
2 parents 2dacdf5 + 0888f87
commit a48333e
Show file tree

Hide file tree

Showing 2 changed files with 101 additions and 1 deletion.
diff --git a/bin/AWS/send_log_email.sh b/bin/AWS/send_log_email.sh
@@ -0,0 +1,98 @@
+#!/usr/bin/env bash
+
+# usage function
+function usage()
+{
+   cat << HEREDOC
+
+   Usage: ./send_log_email.sh -f <sender email> -t <recipient email> -s <email subject> -a <attachment path>
+
+   arguments:
+     -f,--sender-email STR        email address of the sender
+     -t,--recipient-email STR     email address of the recipient
+     -s,--subject-text STR        email subject text
+     -a,--attachment-path STR     path to the log file which should be sent as an attachment
+
+   optional arguments:
+     -b,--body-text STR           email body text
+     -n,--filter-tail-n INT       filtering for only last N rows of the logging file. Default is 100.
+     -o,--only-filtered           send only the filtered (last N rows) file as an email attachment to save space
+     -h, --help                   show this help message and exit
+
+HEREDOC
+}
+
+sender_email=''
+recipient_email=''
+subject_text=''
+body_text=''
+
+attachment_path=''
+
+filter_last_lines=100
+send_only_tail_filtered_log=false
+
+
+while [[ $# -gt 0 ]]; do
+key="$1"
+
+case $key in
+    -f|--sender-email)
+    sender_email="$2"
+    shift 2 # past argument value
+    ;;
+    -t|--recipient-email)
+    recipient_email="$2"
+    shift 2 # past argument value
+    ;;
+    -s|--subject-text)
+    subject_text="$2"
+    shift 2 # past argument value
+    ;;
+    -b|--body-text)
+    body_text="$2"
+    shift 2 # past argument value
+    ;;
+    -a|--attachment-path)
+    attachment_path="$2"
+    shift 2 # past argument value
+    ;;
+    -n|--filter-tail-n)
+    filter_last_lines="$2"
+    shift 2 # past argument value
+    ;;
+    -o|--only-filtered)
+    send_only_tail_filtered_log=true
+    shift 1 # past argument value
+    ;;
+    -h|--help )
+    usage;
+    exit;
+    ;;
+    *)    # unknown option
+    echo "Don't know the argument"
+    usage;
+    exit;
+    ;;
+esac
+done
+
+
+if [ "$sender_email" == "" ] || [ "$recipient_email" == "" ] || [ "$subject_text" == "" ] || [ "$attachment_path" == "" ]; then
+    echo "Not provided required parameters"
+    usage
+    exit
+fi
+
+
+attachment_filename=$(basename "$attachment_path")
+
+if [ "$send_only_tail_filtered_log" == true ]; then
+    echo '{"Data": "From: '${sender_email}'\nTo: '${recipient_email}'\nSubject: '${subject_text}'\nMIME-Version: 1.0\nContent-type: Multipart/Mixed; boundary=\"NextPart\"\n\n--NextPart\nContent-Type: text/plain\n\n'${body_text}'\n\n--NextPart\nContent-Type: text/plain;\nContent-Transfer-Encoding: base64;\nContent-Disposition: attachment; filename=\"tail_'${attachment_filename}'\"\n\n'$(tail -n $filter_last_lines $attachment_path | base64)'\n\n--NextPart--"}' > ~/log_email_message.json
+else
+    echo '{"Data": "From: '${sender_email}'\nTo: '${recipient_email}'\nSubject: '${subject_text}'\nMIME-Version: 1.0\nContent-type: Multipart/Mixed; boundary=\"NextPart\"\n\n--NextPart\nContent-Type: text/plain\n\n'${body_text}'\n\n--NextPart\nContent-Type: text/plain;\nContent-Transfer-Encoding: base64;\nContent-Disposition: attachment; filename=\"'${attachment_filename}'\"\n\n'$(base64 $attachment_path)'\n\n--NextPart\nContent-Type: text/plain;\nContent-Transfer-Encoding: base64;\nContent-Disposition: attachment; filename=\"tail_'${attachment_filename}'\"\n\n'$(tail -n $filter_last_lines $attachment_path | base64)'\n\n--NextPart--"}' > ~/log_email_message.json
+fi
+
+aws ses send-raw-email --region eu-west-1 --raw-message file://~/log_email_message.json
+
+rm ~/log_email_message.json
diff --git a/bin/AWS/test_core_pytorch_compare.sh b/bin/AWS/test_core_pytorch_compare.sh
@@ -46,6 +46,7 @@ HEREDOC
 }
 
 key_path=$(jq -r '.key_path' configs/my_config.json)
+email_address=$(jq -r '.email' configs/my_config.json)
 instance_config="default_config.json"
 instance_type=
 username="ubuntu"
@@ -159,6 +160,7 @@ ssh -i $key_path -o "StrictHostKeyChecking no" $username@$ec2_instance_address '
 scp -i $key_path -r ../../aitoolbox $username@$ec2_instance_address:~/package_test
 scp -i $key_path -r ../../tests_gpu $username@$ec2_instance_address:~/package_test
 scp -i $key_path ../../requirements.txt $username@$ec2_instance_address:~/package_test
+scp -i $key_path send_log_email.sh $username@$ec2_instance_address:~
 
 terminate_setting=""
 if [ "$debug_mode" == false ]; then
@@ -170,7 +172,7 @@ fi
 #########################################################
 echo "Running the comparison tests"
 ssh -i $key_path $username@$ec2_instance_address \
-    "source activate $py_env ; tmux new-session -d -s 'training' 'export AWS_DEFAULT_REGION=$aws_region ; cd package_test ; pip install pytest datasets ; pip install -r requirements.txt ; python -m pytest $pytest_dir -s ; aws s3 cp $logging_path s3://aitoolbox-testing/core_pytorch_comparisson_testing/$logging_filename ; $terminate_setting' \; pipe-pane 'cat > $logging_path'"
+    "source activate $py_env ; tmux new-session -d -s 'training' 'export AWS_DEFAULT_REGION=$aws_region ; cd package_test ; pip install pytest datasets ; pip install -r requirements.txt ; python -m pytest $pytest_dir -s ; aws s3 cp $logging_path s3://aitoolbox-testing/core_pytorch_comparisson_testing/$logging_filename ; ~/send_log_email.sh -f $email_address -t $email_address -s \"GPU testing results for: $pytest_dir\" --attachment-path $logging_path ; $terminate_setting' \; pipe-pane 'cat > $logging_path'"
 
 echo "Instance IP: $ec2_instance_address"