From 0f4f50e2948fa56770ee112183ab9c25e0150caa Mon Sep 17 00:00:00 2001 From: Blake Caldwell Date: Mon, 11 May 2020 16:39:08 -0400 Subject: [PATCH 1/8] MAINT: docker image: hnn_out permissions check --- installer/docker/check_hnn_out_perms.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/installer/docker/check_hnn_out_perms.sh b/installer/docker/check_hnn_out_perms.sh index f2ae909ea..47620daa5 100755 --- a/installer/docker/check_hnn_out_perms.sh +++ b/installer/docker/check_hnn_out_perms.sh @@ -13,11 +13,11 @@ for dir in "${SYSTEM_USER_DIR}/hnn_out" "${SYSTEM_USER_DIR}/hnn_out/data" "${SYS mkdir "$dir" fi if [[ $? -ne 0 ]]; then - echo "failed" - echo "Error: $dir has incorrect permissions or could not create $dir. Can't start HNN." + echo "*failed*" + echo "Error: $dir has incorrect permissions or could not create $dir. Output from ls -l:" ls -ld "$dir" exit 1 fi done -echo "ok" +echo "done" From 3dccaa8eccac96a6d5b7ff83663cfe78376cea82 Mon Sep 17 00:00:00 2001 From: Blake Caldwell Date: Thu, 7 May 2020 16:24:06 -0400 Subject: [PATCH 2/8] MAINT: another refactor of hnn_docker.sh Lots of updates to the docker run commands for starting the container under different OS platforms. Travis now installs WSL, but tests aren't ready yet. --- hnn_docker.sh | 113 +++-- installer/windows/Dockerfile | 5 +- scripts/docker_functions.sh | 923 ++++++++++++++++++++++++----------- 3 files changed, 715 insertions(+), 326 deletions(-) diff --git a/hnn_docker.sh b/hnn_docker.sh index a5c3eb5c3..7e174551e 100755 --- a/hnn_docker.sh +++ b/hnn_docker.sh @@ -10,9 +10,14 @@ cleanup 1 } +[[ $LOGFILE ]] || export LOGFILE="$(pwd)/hnn_docker.log" source scripts/docker_functions.sh -export LOGFILE="$(pwd)/hnn_docker.log" + set_globals +if [[ $? -ne 0 ]]; then + echo "*failed*" | tee -a "$LOGFILE" + cleanup 2 +fi echo -e "======================================" while [ -n "$1" ]; do @@ -41,7 +46,7 @@ print_header_message "Checking OS version... " echo "$OS" | tee -a "$LOGFILE" # defaults -if [[ "$OS" =~ "linux" ]]; then +if [[ "$OS" == "linux" ]] || [[ "$OS" == "wsl" ]]; then [[ $USE_SSH ]] || USE_SSH=0 else [[ $USE_SSH ]] || USE_SSH=1 @@ -77,19 +82,25 @@ if [[ $? -ne "0" ]]; then fi fi -if [[ $UPGRADE -eq "1" ]]; then +get_docker_container_type +if [[ $? -ne "0" ]]; then + exit $? +fi +# CONTAINER_TYPE is set + +if [[ $UPGRADE -eq 1 ]]; then retry_docker_pull - if [[ $? -ne "0" ]]; then + if [[ $? -ne 0 ]]; then cleanup 2 fi - if [[ $START -eq "0" ]]; then + if [[ $START -eq 0 ]]; then # just doing upgrade cleanup 0 fi fi -if [[ "$STOP" -eq "1" ]]; then +if [[ $STOP -eq 1 ]]; then stop_container_fail if [[ "${DOCKER_TOOLBOX}" -eq "1" ]]; then stop_docker_machine_fail @@ -128,15 +139,11 @@ set_xauthority # can use global XAUTHORITY # start X server and set DISPLAY -if [[ "$OS" =~ "windows" ]]; then +if [[ "$OS" == "windows" ]] || [[ "$OS" == "wsl" ]]; then check_vcxsrv_running_print if [[ $? -ne 0 ]]; then find_command_suggested_path "vcxsrv" "/c/Program Files/VcXsrv" - if [[ $? -eq 0 ]]; then - # duplicate the functionality of the 'basename' command - # (remove the part between the last '/' and the end of line '$') - VCXSRV_DIR="$(echo ${vcxsrv_cmd}|sed 's/\/[^\/]\{1,\}$//')" - else + if [[ $? -ne 0 ]]; then echo "Could not find vcxsrv command. Please run XLaunch manually" | tee -a "$LOGFILE" cleanup 1 fi @@ -145,10 +152,18 @@ if [[ "$OS" =~ "windows" ]]; then set_local_display_from_port 0 # DISPLAY is set - find_command_suggested_path "xauth" "/c/Program Files/VcXsrv" - if [[ $? -ne 0 ]]; then - echo "Could not find xauth command. Please check VcXsrv installation" | tee -a "$LOGFILE" - cleanup 1 + if [[ "$OS" == "windows" ]]; then + find_command_suggested_path "xauth" "/c/Program Files/VcXsrv" + if [[ $? -ne 0 ]]; then + echo "Could not find xauth command. Please check VcXsrv installation" | tee -a "$LOGFILE" + cleanup 1 + fi + else + find_program_print xauth + if [[ $? -ne 0 ]]; then + echo "Could not find xauth command." | tee -a "$LOGFILE" + cleanup 1 + fi fi set_local_display_from_port 0 @@ -213,7 +228,7 @@ fi # ignore hostname in Xauthority by setting FamilyWild mask # https://stackoverflow.com/questions/16296753/can-you-run-gui-applications-in-a-docker-container/25280523#25280523 -KEYS_TO_CONVERT=$(echo "$(get_xauth_keys)" | grep -v '^ffff') # we know get_xauth_keys will succeed after above +KEYS_TO_CONVERT=$(echo "$(get_host_xauth_keys)" | grep -v '^ffff') # we know get_xauth_keys will succeed after above if [[ -n $KEYS_TO_CONVERT ]]; then print_header_message "Updating xauth keys for use with docker... " # this command is too complicated to try to wrap in a function... quoting all variables @@ -238,7 +253,7 @@ fi check_hnn_out_perms_host if [[ $? -ne "0" ]]; then - create_hnn_out_fail + fix_hnn_out_perms_host_fail fi echo | tee -a "$LOGFILE" @@ -246,7 +261,7 @@ echo "Starting HNN" | tee -a "$LOGFILE" echo "--------------------------------------" | tee -a "$LOGFILE" # check if container exists -output_existing_container_command > /dev/null +output_existing_container_command &> /dev/null if [[ $? -eq "0" ]]; then # container does exist, so check if it is running print_header_message "Checking for running HNN container... " @@ -257,6 +272,23 @@ if [[ $? -eq "0" ]]; then else ALREADY_RUNNING=0 echo "not found" | tee -a "$LOGFILE" + print_header_message "Starting existing HNN container... " + start_container_silent + if [[ $? -eq 0 ]]; then + echo "done" | tee -a "$LOGFILE" + else + echo "*failed*" | tee -a "$LOGFILE" + if [[ $TRAVIS_TESTING -eq 1 ]]; then + find_existing_container_print + if [[ $? -eq "0" ]]; then + print_header_message "Removing old container... " + remove_container_fail + fi + else + echo "Removing old container might resolve failure. Verification required." | tee -a "$LOGFILE" + prompt_remove_container_fail + fi + fi fi else ALREADY_RUNNING=0 @@ -264,30 +296,33 @@ fi if [[ $ALREADY_RUNNING -eq 0 ]]; then # start the container - docker_run_print - if [[ $? -ne 0 ]]; then - RETRY=1 - # try starting again - docker_run_print - if [[ $? -ne 0 ]]; then - cleanup 2 - fi - fi + docker_run_fail # container is up - # Workaround because in the qemu docker-machine (used with mac and windows). - # volume mounts don't work (hnn_out and hnn_source_code). - if [[ $TRAVIS_TESTING -eq 1 ]] && [[ ! "$OS" =~ "linux" ]]; then + if [[ $USE_SSH -eq 1 ]]; then + copy_ssh_files_to_container_fail + fi + + if [[ $TRAVIS_TESTING -eq 1 ]]; then + # Use Travis's checkout of the code inside the container copy_hnn_source_fail + copy_hnn_out_fail fi fi -if [[ $TRAVIS_TESTING -eq 0 ]] || ([[ ! "$OS" =~ "windows" ]] && [[ ! "$OS" =~ "wsl" ]]); then +if [[ $TRAVIS_TESTING -eq 1 ]]; then + # we may not have write access if the user changed (i.e. now USE_SSH=1) + change_hnn_source_perms_fail +fi + +fix_hnn_out_perms_container_print +if [[ $? -ne 0 ]]; then + fix_hnn_out_perms_host_fail +fi + +if [[ "$CONTAINER_TYPE" == "linux" ]]; then check_x_port_container_fail # could open port - - check_hnn_out_perms_container_fail - # can read/write to hnn_out directory fi setup_xauthority_in_container_fail @@ -302,9 +337,9 @@ if [[ $USE_SSH -eq 0 ]]; then cleanup 2 fi else - if [[ $TRAVIS_TESTING -eq 1 ]] && ([[ "$OS" =~ "windows" ]] || [[ "$OS" == "wsl" ]]); then + if [[ "$CONTAINER_TYPE" == "windows" ]]; then echo "SSH mode not supported with windows containers" - cleanup 2 + cleanup 1 fi if [[ $ALREADY_RUNNING -eq 0 ]]; then # need to start sshd @@ -323,7 +358,7 @@ else ssh_start_hnn_print if [[ $? -ne 0 ]]; then # failure could be caused by bad ssh keys - copy_ssh_files_to_running_container_fail + copy_ssh_files_to_container_fail RETRY=1 DEBUG=1 @@ -336,7 +371,7 @@ else fi if [[ $TRAVIS_TESTING -eq 1 ]]; then - echo "Testing mode: "/home/$TEST_USER/hnn/hnn_docker.log"" | tee -a "$LOGFILE" + echo "Testing mode: $LOGFILE" | tee -a "$LOGFILE" fi ## DONE! all successful cases reach here diff --git a/installer/windows/Dockerfile b/installer/windows/Dockerfile index 50cdf1c5b..a77391d43 100644 --- a/installer/windows/Dockerfile +++ b/installer/windows/Dockerfile @@ -1,4 +1,4 @@ -FROM mcr.microsoft.com/windows/servercore:ltsc2019 +FROM mcr.microsoft.com/windows/servercore:1909 # SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue';"] SHELL ["cmd", "/S", "/C"] @@ -38,7 +38,8 @@ RUN powershell -executionpolicy bypass -File "C:\home\hnn_user\Downloads\hnn.ps1 del home\hnn_user\Downloads\*.* /Q RUN cd home\hnn_user && \ - move /y hnn hnn_source_code + move /y hnn hnn_source_code && \ + mkdir .ssh WORKDIR /home/hnn_user/hnn_source_code diff --git a/scripts/docker_functions.sh b/scripts/docker_functions.sh index 5fee2cdfa..bc5964767 100644 --- a/scripts/docker_functions.sh +++ b/scripts/docker_functions.sh @@ -38,6 +38,45 @@ function check_var { fi } +function get_wsl_home_path { + # no arguments + check_var ESC_STR + + local __command + local __index + local __arg + local __send_args + local __home + + log_header_message "Getting path outside of WSL... " + __command=("powershell.exe" "\$env:HOME" "tr" "-d '\r'") + let __index=0 + for __arg in "${__command[@]}"; do + __send_args[$__index]=$(echo $__arg|sed "s/ /$ESC_STR/g") + (( __index++ )) + done + __home=$(output_run_piped_command "${__send_args[@]}") + if [[ $? -ne 0 ]]; then + echo "*failed*" >> "$LOGFILE" + return 2 + else + echo "done" >> "$LOGFILE" + fi + + log_header_message "Turning into UNIX path... " + COMMAND=(3 "cygpath.exe" "-u" "$__home") + convert_COMMAND_to_escaped_array + __home=$(output_run_command_arguments "${ESCAPED_COMMAND[@]}") + if [[ $? -ne 0 ]]; then + echo "*failed*" >> "$LOGFILE" + return 2 + else + echo "done" >> "$LOGFILE" + fi + + echo "$__home" +} + function get_os { OS_OUTPUT=$(uname -a) if [[ $OS_OUTPUT =~ "MINGW" ]] || [[ $OS_OUTPUT =~ "MSYS" ]]; then @@ -68,34 +107,38 @@ function set_globals { SSHD_STARTED=0 NEW_XAUTH_KEYS=0 ESC_STR="%^%" + VCXSRV_PID= + + SSH_PRIVKEY="$(pwd)/installer/docker/id_rsa_hnn" + SSH_PUBKEY="$(pwd)/installer/docker/id_rsa_hnn.pub" + SSH_AUTHKEYS="$(pwd)/installer/docker/authorized_keys" + + export USE_SSH UPGRADE STOP START RETRY UNINSTALL HNN_DOCKER_IMAGE HNN_CONTAINER_NAME + export SYSTEM_USER_DIR ALREADY_RUNNING SSHD_STARTED NEW_XAUTH_KEYS ESC_STR VCXSRV_PID + OS=$(get_os) __command_status=$? if [[ $__command_status -ne 0 ]]; then echo "Failed to get OS type" | tee -a "$LOGFILE" - exit $__command_status + return $__command_status fi if [[ "$OS" == "wsl" ]]; then - echo "WSL is not supported yet" - # exit 1 + export SYSTEM_USER_DIR="$(get_wsl_home_path)" + if [[ $? -ne 0 ]]; then + return $? + fi set_local_display_from_port 0 fi # if display hasn't been set yet, give it a default value for the OS [[ "$DISPLAY" ]] || set_local_display_from_port 0 - SSH_PRIVKEY="$(pwd)/installer/docker/id_rsa_hnn" - SSH_PUBKEY="$(pwd)/installer/docker/id_rsa_hnn.pub" - SSH_AUTHKEYS="$(pwd)/installer/docker/authorized_keys" - if [[ "$OS" == "mac" ]]; then function timeout { perl -e 'alarm shift; exec @ARGV' "$@" } export timeout fi - - export USE_SSH UPGRADE STOP START RETRY UNINSTALL HNN_DOCKER_IMAGE HNN_CONTAINER_NAME - export SYSTEM_USER_DIR ALREADY_RUNNING SSHD_STARTED NEW_XAUTH_KEYS ESC_STR } function errexit() { @@ -134,10 +177,10 @@ function cleanup { echo "cleanup() called from: ${FUNCNAME[1]} (L:${BASH_LINENO[0]})" >> "$LOGFILE" if [ ! -z "${VCXSRV_PID}" ]; then stop_vcxsrv - # if [[ $? -ne 0 ]]; then - echo "Killing VcXsrv PID ${VCXSRV_PID}" >> "$LOGFILE" - kill ${VCXSRV_PID} &> /dev/null - # fi + if [[ $? -ne 0 ]]; then + echo "Killing VcXsrv PID ${VCXSRV_PID}" >> "$LOGFILE" + kill ${VCXSRV_PID} &> /dev/null + fi fi if [[ $__failed -eq "0" ]]; then @@ -226,7 +269,7 @@ function output_run_piped_command { echo -e "\n ** Command: $__binary1 $__command_args1 | $__binary2 $__command_args2" >> "$LOGFILE" echo -n " ** Stderr: " >> "$LOGFILE" - __output=$("$__binary1" $__command_args1 | "$__binary2" $__command_args2 2>> "$LOGFILE") + __output=$("$__binary1" $__command_args1 2>> "$LOGFILE" | "$__binary2" $__command_args2 2>> "$LOGFILE") __command_status=$? if [[ ! -z "$__output" ]]; then echo -e "\n ** Stdout: $__output" | tr -d '\r' >> "$LOGFILE" @@ -265,8 +308,8 @@ function output_run_command_arguments { # note that output here is not expected to go to stdout, so redirect to LOGFILE re='^[0-9]+$' if ! [[ $1 =~ $re ]] ; then - echo -e "\nError: $FUNCNAME argument $1 is not a number" >> "$LOGFILE" - cleanup 1 + echo -e "\nError: $FUNCNAME: argument '$1' is not a number" >> "$LOGFILE" + exit 1 fi check_var LOGFILE @@ -281,7 +324,7 @@ function output_run_command_arguments { if [[ $__num_args -ne $__expected_num_args ]]; then echo -e "\nError: $FUNCNAME (L:${BASH_LINENO[0]}) must have $__expected_num_args arguments" >> "$LOGFILE" echo "Got $__num_args arguments: $@" >> "$LOGFILE" - cleanup 1 + exit 1 fi local __index @@ -322,11 +365,16 @@ function output_run_command_arguments { __command_status=$? if [[ -n "$__output" ]]; then echo >> "$LOGFILE" - echo " ** Stdout: $__output" | tr -d '\r' >> "$LOGFILE" + if [[ $SUPPRESS_OUTPUT -eq 1 ]]; then + echo " ** Stdout: ** suppresed **" >> "$LOGFILE" + SUPPRESS_OUTPUT=1 + else + echo " ** Stdout: $__output" | tr -d '\r' >> "$LOGFILE" + fi fi # send output back to caller - echo "$__output" + echo "$__output" | tr -d '\r' if [[ $__command_status -eq 0 ]]; then true @@ -420,7 +468,16 @@ function remove_container_fail { check_var docker_cmd check_var HNN_CONTAINER_NAME - run_command_print_status_failure_exit "$docker_cmd rm -fv $HNN_CONTAINER_NAME" + COMMAND=(4 "$docker_cmd" "rm" "-fv" "$HNN_CONTAINER_NAME") + convert_COMMAND_to_escaped_array + output_run_command_arguments "${ESCAPED_COMMAND[@]}" &> /dev/null + if [[ $? -ne "0" ]]; then + echo "*failed*" | tee -a "$LOGFILE" + echo -e "\nFailure to remove container can sometimes be fixed by restarting Docker" + cleanup 2 + else + echo "done" | tee -a "$LOGFILE" + fi } function print_header_message { @@ -442,20 +499,42 @@ function print_header_message_short { echo -n "$1" | tee -a "$LOGFILE" } +function log_header_message { + check_args "$@" $# 1 + check_var LOGFILE + + # first arg is message to print. surrounded by newlines in "$LOGFILE" + echo >> "$LOGFILE" + echo -n "$1" >> "$LOGFILE" + echo >> "$LOGFILE" +} + function start_vcxsrv_print { check_var LOGFILE - check_var VCXSRV_DIR + check_var vcxsrv_cmd - print_header_message "Starting VcXsrv... " - echo -e "\n ** Command: ${VCXSRV_DIR}/vcxsrv.exe -wgl -multiwindow 2>&1 &" >> "$LOGFILE" - if [[ $DEBUG -eq 1 ]] || [[ $VERBOSE -eq 1 ]]; then - echo -n " ** Output: " >> "$LOGFILE" - "${VCXSRV_DIR}/vcxsrv.exe" -wgl -multiwindow >> "$LOGFILE" 2>&1 & - else - "${VCXSRV_DIR}/vcxsrv.exe" -wgl -multiwindow > /dev/null 2>&1 & + local __vcxsrv_dir + + # duplicate the functionality of the 'basename' command + # (remove the part between the last '/' and the end of line '$') + __vcxsrv_dir="$(echo ${vcxsrv_cmd}|sed 's/\/[^\/]\{1,\}$//')" + if [[ ! -d "$__vcxsrv_dir" ]]; then + echo "Could not find directory for ${vcxsrv_cmd}" + cleanup 2 fi + + print_header_message "Starting VcXsrv... " + COMMAND=(3 "${__vcxsrv_dir}/vcxsrv.exe" "-wgl" "-multiwindow") + convert_COMMAND_to_escaped_array + output_run_command_arguments "${ESCAPED_COMMAND[@]}" &> /dev/null & VCXSRV_PID=$! - echo "done" | tee -a "$LOGFILE" + + log_header_message "Checking if VcXsrv is running..." + COMMAND=(3 "kill" "-0" "$VCXSRV_PID") + convert_COMMAND_to_escaped_array + output_run_command_arguments "${ESCAPED_COMMAND[@]}" &> /dev/null + fail_on_bad_exit $? + echo "Started VcXsrv with PID ${VCXSRV_PID}" >> "$LOGFILE" } @@ -465,7 +544,11 @@ function stop_vcxsrv { print_header_message "Stopping VcXsrv... " echo >> "$LOGFILE" - run_command_print_status "cmd.exe //c taskkill //F //IM vcxsrv.exe" + if [[ "$OS" == "wsl" ]]; then + run_command_print_status "cmd.exe /c taskkill /F /IM vcxsrv.exe" + else + run_command_print_status "cmd.exe //c taskkill //F //IM vcxsrv.exe" + fi if [[ $? -eq "0" ]]; then VCXSRV_PID= else @@ -516,6 +599,7 @@ function get_container_port { check_var LOGFILE check_var HNN_CONTAINER_NAME check_var docker_cmd + check_var TRAVIS_TESTING local __index local __send_args @@ -529,6 +613,7 @@ function get_container_port { # this is hardcoded in docker-machine-driver-qemu __ssh_port="5000" else + log_header_message "Getting port that maps to docker container port 22... " __command=("$docker_cmd" "port $HNN_CONTAINER_NAME 22") let __index=0 for __arg in "${__command[@]}"; do @@ -538,6 +623,7 @@ function get_container_port { __port_string=$(output_run_specific_command "${__send_args[@]}") if [[ $? -ne "0" ]]; then + echo "*failed*" >> "$LOGFILE" echo "failed to run ${__command[@]}" >> "$LOGFILE" return 1 fi @@ -545,9 +631,11 @@ function get_container_port { __ssh_port=$(echo $__port_string| cut -d':' -f 2) re='^[0-9]+$' if ! [[ $__ssh_port =~ $re ]] ; then + echo "*failed*" >> "$LOGFILE" echo "failed to get a port number from \"$__ssh_port\"" >> "$LOGFILE" return 1 fi + echo "done" >> "$LOGFILE" fi echo $__ssh_port @@ -565,6 +653,8 @@ function ssh_start_hnn_print { check_var DEBUG check_var VERBOSE check_var LOGFILE + check_var CONTAINER_TYPE + check_var TRAVIS_TESTING local __verbose local __ssh_port @@ -592,9 +682,13 @@ function ssh_start_hnn_print { # since we assigned port 6000, we can be certain of this DISPLAY export DISPLAY=127.0.0.1:0 - export XAUTHORITY="/tmp/.Xauthority" export TRAVIS_TESTING - export SYSTEM_USER_DIR + + if [[ "$CONTAINER_TYPE" == "windows" ]]; then + export XAUTHORITY="/c/home/hnn_user/.Xauthority" + else + export XAUTHORITY="/tmp/.Xauthority" + fi if [[ $RETRY -eq 1 ]]; then echo -n "(retry) " | tee -a "$LOGFILE" @@ -612,6 +706,7 @@ function ssh_start_hnn_print { # server is running) and port 6000 in the container we are ssh'ing into. Other options are to avoid # warnings about hostkeychecking and to not prompt for a password if public key authentication fails. print_header_message "Starting HNN GUI... " + COMMAND=(23 "ssh" "-o" "SendEnv=DISPLAY" "-o" "SendEnv=XAUTHORITY" "-o" "SendEnv=SYSTEM_USER_DIR" "-o" "SendEnv=TRAVIS_TESTING" \ "-o" "PasswordAuthentication=no" "-o" "UserKnownHostsFile=/dev/null" "-o" "StrictHostKeyChecking=no" \ "$__verbose" "-i" "$SSH_PRIVKEY" "-R" "6000:127.0.0.1:6000" "hnn_user@$__docker_host_ip" "-p" "$__ssh_port") @@ -630,16 +725,18 @@ function start_hnn_print { # runs start_hnn.sh script directly in container using "docker exec" check_var docker_cmd check_var TRAVIS_TESTING + check_var SYSTEM_USER_DIR check_var HNN_CONTAINER_NAME check_var LOGFILE + check_var CONTAINER_TYPE + check_var UID local __display # set DISPLAY for OS + print_header_message "Setting DISPLAY variable for $CONTAINER_TYPE containers... " __display=$(get_display_for_gui) - if [[ $? -ne 0 ]]; then - cleanup $? - fi + fail_on_bad_exit $? if [[ $RETRY -eq 1 ]]; then echo -n "(retry) " | tee -a "$LOGFILE" @@ -647,16 +744,18 @@ function start_hnn_print { fi print_header_message "Starting HNN GUI... " - if [[ $TRAVIS_TESTING -eq 1 ]] && [[ "$OS" =~ "windows" ]]; then - COMMAND=(11 "$docker_cmd" "exec" "--env" "SYSTEM_USER_DIR=$HOME" "--env" "TRAVIS_TESTING=$TRAVIS_TESTING" \ - "--env" "DISPLAY=$__display" "$HNN_CONTAINER_NAME" "bash" "/c/home/hnn_user/start_hnn.sh") - elif [[ $TRAVIS_TESTING -eq 1 ]] && [[ "$OS" == "wsl" ]]; then - COMMAND=(11 "$docker_cmd" "exec" "--env" "SYSTEM_USER_DIR=/$HOME" "--env" "TRAVIS_TESTING=$TRAVIS_TESTING" \ - "--env" "DISPLAY=$__display" "$HNN_CONTAINER_NAME" "bash" "/c/home/hnn_user/start_hnn.sh") + if [[ "$CONTAINER_TYPE" == "windows" ]]; then + __container_user="hnn_user" + __start_script="/c/home/hnn_user/start_hnn.sh" else - COMMAND=(12 "$docker_cmd" "exec" "--env" "SYSTEM_USER_DIR=$HOME" "--env" "TRAVIS_TESTING=$TRAVIS_TESTING" \ - "--env" "DISPLAY=$__display" "-u" "$UID" "$HNN_CONTAINER_NAME" "/home/hnn_user/start_hnn.sh") + __container_user="$UID" + __start_script="/home/hnn_user/start_hnn.sh" fi + COMMAND=(13 "$docker_cmd" "exec" "--env" "TRAVIS_TESTING=$TRAVIS_TESTING" \ + "--env" "SYSTEM_USER_DIR=$SYSTEM_USER_DIR" "--env" "DISPLAY=$__display" \ + "-u" "$__container_user" "$HNN_CONTAINER_NAME" \ + "bash" "$__start_script") + convert_COMMAND_to_escaped_array MSYS_NO_PATHCONV=1 output_run_command_arguments "${ESCAPED_COMMAND[@]}" &> /dev/null if [[ $? -ne "0" ]]; then @@ -684,15 +783,30 @@ function check_sshd_proc { MSYS_NO_PATHCONV=1 run_command_print_status "$docker_cmd exec $HNN_CONTAINER_NAME pgrep sshd" } -function create_hnn_out_fail { +function fix_hnn_out_perms_host_fail { # no arguments check_var HOME + check_var TRAVIS_TESTING - print_header_message "Creating ${HOME}/hnn_out... " - COMMAND=(2 "mkdir" "${HOME}/hnn_out") - convert_COMMAND_to_escaped_array - output_run_command_arguments "${ESCAPED_COMMAND[@]}" &> /dev/null + if [[ ! -e "${SYSTEM_USER_DIR}/hnn_out/" ]]; then + print_header_message "Creating ${HOME}/hnn_out... " + COMMAND=(2 "mkdir" "${HOME}/hnn_out") + convert_COMMAND_to_escaped_array + output_run_command_arguments "${ESCAPED_COMMAND[@]}" &> /dev/null + fail_on_bad_exit $? + return + fi + + print_header_message "Updating hnn_out permissions on host... " + find "${SYSTEM_USER_DIR}/hnn_out" -type d -exec chmod 777 {} \; >> "$LOGFILE" 2>&1 && \ + find "${SYSTEM_USER_DIR}/hnn_out" -type f -exec chmod 666 {} \; >> "$LOGFILE" 2>&1 fail_on_bad_exit $? + + if [[ $TRAVIS_TESTING -eq 1 ]] && [[ "$OS" == "mac" ]]; then + COMMAND=(2 "touch" "${SYSTEM_USER_DIR}/hnn_out/THIS_DIRECTORY_IS_SHARED_BETWEEN_DOCKER_AND_YOUR_OS") + convert_COMMAND_to_escaped_array + output_run_command_arguments "${ESCAPED_COMMAND[@]}" &> /dev/null + fi } function check_hnn_out_perms_host { @@ -700,62 +814,77 @@ function check_hnn_out_perms_host { check_var LOGFILE check_var HOME - COMMAND=(3 "test" "-w" "${HOME}/hnn_out") - convert_COMMAND_to_escaped_array - output_run_command_arguments "${ESCAPED_COMMAND[@]}" &> /dev/null + installer/docker/check_hnn_out_perms.sh if [[ $? -ne 0 ]]; then return 1 fi } -function check_hnn_out_perms_container_fail { +function check_hnn_out_perms_container_print { # no arguments # runs script in container that checks permissions for hnn_out are okay check_var docker_cmd check_var HNN_CONTAINER_NAME - check_var TRAVIS_TESTING + check_var CONTAINER_TYPE check_var LOGFILE check_var RETRY + check_var SYSTEM_USER_DIR + + local __script_path + local __user + if [[ $RETRY -eq 1 ]]; then echo -n "(retry) " | tee -a "$LOGFILE" RETRY=0 fi - print_header_message "Checking permissions of ${HOME}/hnn_out in container... " - COMMAND=(6 "$docker_cmd" "exec" "--env" "SYSTEM_USER_DIR=$HOME" "$HNN_CONTAINER_NAME" "/home/hnn_user/check_hnn_out_perms.sh") + print_header_message "Checking permissions of ${SYSTEM_USER_DIR}/hnn_out in container... " + if [[ "$CONTAINER_TYPE" == "windows" ]]; then + __script_path="bash -c /c/home/hnn_user/check_hnn_out_perms.sh" + else + __script_path="/home/hnn_user/check_hnn_out_perms.sh" + fi + COMMAND=(6 "$docker_cmd" "exec" "$HNN_CONTAINER_NAME" "bash" "-c" "$__script_path") convert_COMMAND_to_escaped_array MSYS_NO_PATHCONV=1 output_run_command_arguments "${ESCAPED_COMMAND[@]}" &> /dev/null if [[ $? -ne 0 ]]; then echo "*failed*" | tee -a "$LOGFILE" - check_hnn_out_perms_host - if [[ $? -ne 0 ]]; then - if [[ "$OS" == "linux" ]]; then - print_header_message "Updating permissions of hnn_out... " - find "$HOME/hnn_out" -type d -exec chmod o+rwx {} \; >> "$LOGFILE" 2>&1 && \ - find "$HOME/hnn_out" -type f -exec chmod o+rw {} \; >> "$LOGFILE" 2>&1 - echo "done" | tee -a "$LOGFILE" - else - echo "Please make ${HOME}/hnn_out accessible by the user running docker (try making world readable/writable)" | tee -a "$LOGFILE" - fi - else - echo "done" | tee -a "$LOGFILE" - echo -e "\nFailure seems to be an issue with docker container." | tee -a "$LOGFILE" - echo "Please open an issue on github with $LOGFILE" | tee -a "$LOGFILE" - echo "https://github.com/jonescompneurolab/hnn/issues" | tee -a "$LOGFILE" - fi - echo "*failed*" | tee -a "$LOGFILE" - cleanup 2 + false else - echo "done" | tee -a "$LOGFILE" + echo "done" fi +} - if [[ $TRAVIS_TESTING -ne 0 ]] || [[ "$OS" == "linux" ]]; then - # This command will not work when a qemu VM is used with docker-machine. - # That is, only when TRAVIS_TESTING=1 and (OS="mac" or OS="windows") - COMMAND=(2 "touch" "$HOME/hnn_out/THIS_DIRECTORY_IS_SHARED_BETWEEN_DOCKER_AND_YOUR_OS") +function fix_hnn_out_perms_container_print { + # no arguments + # runs command in container to change permissions of hnn_out + check_var docker_cmd + check_var HNN_CONTAINER_NAME + check_var CONTAINER_TYPE + check_var LOGFILE + check_var SYSTEM_USER_DIR + + local __container_user + + if [[ "$CONTAINER_TYPE" == "windows" ]]; then + return 0 + else + __container_user="root" + fi + + check_hnn_out_perms_container_print + if [[ $? -ne 0 ]]; then + print_header_message "Updating hnn_out permissions inside container... " + COMMAND=(8 "$docker_cmd" "exec" "-u" "$__container_user" "$HNN_CONTAINER_NAME" "bash" "-c" \ + "find \"${SYSTEM_USER_DIR}/hnn_out\" -type d -exec chmod 777 {} \; && \ + find \"${SYSTEM_USER_DIR}/hnn_out\" -type f -exec chmod 666 {} \;") convert_COMMAND_to_escaped_array - output_run_command_arguments "${ESCAPED_COMMAND[@]}" &> /dev/null + MSYS_NO_PATHCONV=1 output_run_command_arguments "${ESCAPED_COMMAND[@]}" &> /dev/null + fail_on_bad_exit $? + + RETRY=1 + check_hnn_out_perms_container_print fi } @@ -763,6 +892,14 @@ function check_x_authenticated { # no arguments check_var DISPLAY check_var OS + check_var LOGFILE + + if [[ "$OS" == "wsl" ]]; then + find_program_print xset + if [[ $? -ne 0 ]]; then + echo -e "Skipping X server test: install xset with 'sudo apt-get install x11-xserver-utils'\n" | tee -a "$LOGFILE" + fi + fi if [[ ! "$OS" == "windows" ]]; then print_header_message "Checking if we can authenticate with X server... " @@ -809,7 +946,7 @@ function check_x_port_netcat { __send_args[$__index]=$(echo $__arg|sed "s/ /$ESC_STR/g") (( __index++ )) done - output_run_piped_command "${__send_args[@]}" > /dev/null + output_run_piped_command "${__send_args[@]}" >> "$LOGFILE" false fi } @@ -826,7 +963,7 @@ function check_x_port_container_fail { # set DISPLAY for OS __display=$(get_display_for_gui) if [[ $? -ne 0 ]]; then - cleanup $? + cleanup 2 fi print_header_message "Checking if X server is reachable from container... " @@ -959,7 +1096,10 @@ function copy_xauthority_file_fail { check_var XAUTHORITY check_var HNN_CONTAINER_NAME check_var LOGFILE + check_var CONTAINER_TYPE + local __container_xauthority_path + local __xauthority_path local __command_status local __user __user="$1" @@ -969,24 +1109,38 @@ function copy_xauthority_file_fail { cleanup 2 fi - print_header_message_short "Copying Xauthority file into container... " - if [[ $TRAVIS_TESTING -eq 1 ]] && ([[ "$OS" =~ "windows" ]] || [[ "$OS" == "wsl" ]]); then + __container_xauthority_path="/tmp/.Xauthority" + __xauthority_path="$XAUTHORITY" + + # exceptions for windows containers + if [[ "$CONTAINER_TYPE" == "windows" ]]; then + __container_xauthority_path="/home/hnn_user/.Xauthority" + + if [[ ! "$OS" == "wsl" ]]; then + COMMAND=(3 "cygpath.exe" "-w" "$__xauthority_path") + convert_COMMAND_to_escaped_array + __xauthority_path=$(output_run_command_arguments "${ESCAPED_COMMAND[@]}") + if [[ $? -ne 0 ]]; then + cleanup 2 + fi + fi + stop_container_silent - COMMAND=(4 "$docker_cmd" "cp" "$XAUTHORITY" "$HNN_CONTAINER_NAME:/windows/temp/.Xauthority") - else - COMMAND=(4 "$docker_cmd" "cp" "$XAUTHORITY" "$HNN_CONTAINER_NAME:/tmp/.Xauthority") fi + + print_header_message_short "Copying Xauthority file into container... " + COMMAND=(4 "$docker_cmd" "cp" "$__xauthority_path" "$HNN_CONTAINER_NAME:$__container_xauthority_path") convert_COMMAND_to_escaped_array - output_run_command_arguments "${ESCAPED_COMMAND[@]}" &> /dev/null + MSYS_NO_PATHCONV=1 output_run_command_arguments "${ESCAPED_COMMAND[@]}" &> /dev/null fail_on_bad_exit $? - if [[ $TRAVIS_TESTING -eq 1 ]] && ([[ "$OS" =~ "windows" ]] || [[ "$OS" == "wsl" ]]); then + if [[ "$CONTAINER_TYPE" == "windows" ]]; then start_container_silent return fi print_header_message "Changing Xauthority permissions in container... " COMMAND=(8 "$docker_cmd" "exec" "-u" "root" "$HNN_CONTAINER_NAME" "bash" "-c" \ - "chown $__user:hnn_group /tmp/.Xauthority && chmod g+rw /tmp/.Xauthority") + "chown $__user:hnn_group \"$__container_xauthority_path\" && chmod g+rw \"$__container_xauthority_path\"") convert_COMMAND_to_escaped_array output_run_command_arguments "${ESCAPED_COMMAND[@]}" &> /dev/null fail_on_bad_exit $? @@ -994,89 +1148,208 @@ function copy_xauthority_file_fail { } function copy_hnn_source_fail { - # copies $PWD to hnn_source_code and sets permissions for hnn_out directory - # for the appropriate user + # copies $PWD to hnn_source_code check_var docker_cmd - check_var XAUTHORITY check_var HNN_CONTAINER_NAME - check_var USE_SSH - check_var UID check_var LOGFILE + check_var CONTAINER_TYPE - local __user - - if [[ $USE_SSH -eq 0 ]]; then - __user="$UID" - else - __user="hnn_user" - fi + local __source_code_path + local __current_directory if [[ ! -e "$PWD/hnn.py" ]]; then echo -e "\nBad hnn_source_code directory at $PWD" | tee -a "$LOGFILE" cleanup 2 fi - print_header_message "Removing old hnn_source_code from container... " - if [[ $TRAVIS_TESTING -eq 1 ]] && ([[ "$OS" =~ "windows" ]] || [[ "$OS" == "wsl" ]]); then - COMMAND=(6 "$docker_cmd" "exec" "$HNN_CONTAINER_NAME" "bash" "-c" "rm -rf /home/hnn_user/hnn_source_code") + if [[ "$CONTAINER_TYPE" == "linux" ]]; then + __source_code_path="/home/hnn_user/hnn_source_code" else - COMMAND=(8 "$docker_cmd" "exec" "-u" "root" "$HNN_CONTAINER_NAME" "bash" "-c" "rm -rf /home/hnn_user/hnn_source_code") + __source_code_path="/c/home/hnn_user/hnn_source_code" fi + + print_header_message "Removing old hnn_source_code from container... " + COMMAND=(6 "$docker_cmd" "exec" "$HNN_CONTAINER_NAME" "bash" "-c" "rm -rf $__source_code_path/*") convert_COMMAND_to_escaped_array output_run_command_arguments "${ESCAPED_COMMAND[@]}" &> /dev/null fail_on_bad_exit $? - if [[ $TRAVIS_TESTING -eq 1 ]] && ([[ "$OS" =~ "windows" ]] || [[ "$OS" == "wsl" ]]); then + # exceptions for windows containers + if [[ "$CONTAINER_TYPE" == "windows" ]]; then + COMMAND=(3 "cygpath.exe" "-w" "$PWD") + convert_COMMAND_to_escaped_array + __current_directory=$(output_run_command_arguments "${ESCAPED_COMMAND[@]}") + if [[ $? -ne 0 ]]; then + cleanup 2 + fi + __current_directory="$__current_directory\." + stop_container_silent + else + __current_directory="$PWD/." fi print_header_message_short "Copying hnn_source_code into container... " - COMMAND=(4 "$docker_cmd" "cp" "$PWD" "$HNN_CONTAINER_NAME:/home/hnn_user/hnn_source_code") + COMMAND=(4 "$docker_cmd" "cp" "$__current_directory" "$HNN_CONTAINER_NAME:/home/hnn_user/hnn_source_code/") convert_COMMAND_to_escaped_array - output_run_command_arguments "${ESCAPED_COMMAND[@]}" &> /dev/null + MSYS_NO_PATHCONV=1 output_run_command_arguments "${ESCAPED_COMMAND[@]}" &> /dev/null fail_on_bad_exit $? - if [[ $TRAVIS_TESTING -eq 1 ]] && ([[ "$OS" =~ "windows" ]] || [[ "$OS" == "wsl" ]]); then + if [[ "$CONTAINER_TYPE" == "windows" ]]; then start_container_silent fi +} + +function copy_hnn_out_fail { + # copies $SYSTEM_USER_DIR/hnn_out to hnn_source_code + check_var docker_cmd + check_var HNN_CONTAINER_NAME + check_var LOGFILE + check_var CONTAINER_TYPE + check_var TRAVIS_TESTING - if [[ $TRAVIS_TESTING -eq 0 ]] || ([[ ! "$OS" =~ "windows" ]] && [[ ! "$OS" == "wsl" ]]); then - print_header_message "Changing hnn directory permissions in container... " + local __hnn_out + __hnn_out="$SYSTEM_USER_DIR/hnn_out" + + if [[ ! -e "$__hnn_out" ]]; then + echo -e "\nError: hnn_out doesn't exist at $__hnn_out" | tee -a "$LOGFILE" + cleanup 2 + fi + + if [[ "$OS" == "mac" ]] && [[ $TRAVIS_TESTING -eq 1 ]]; then + print_header_message "Creating hnn_out in container... " COMMAND=(8 "$docker_cmd" "exec" "-u" "root" "$HNN_CONTAINER_NAME" "bash" "-c" \ - "chown -R $__user /home/hnn_user/hnn_source_code && chown $__user $SYSTEM_USER_DIR/hnn_out") + "mkdir -p \"$__hnn_out/data\" && \ + mkdir -p \"$__hnn_out/param\" && \ + chown -R hnn_user:hnn_group \"$__hnn_out\" && \ + chmod 777 -R \"$__hnn_out\"") convert_COMMAND_to_escaped_array output_run_command_arguments "${ESCAPED_COMMAND[@]}" &> /dev/null - fail_on_bad_exit $? + fail_on_bad_exit $? + else + print_header_message "Removing old hnn_out from container... " + + if [[ "$CONTAINER_TYPE" == "windows" ]]; then + COMMAND=(6 "$docker_cmd" "exec" "$HNN_CONTAINER_NAME" "bash" "-c" "rm -rf $__hnn_out/*") + else + COMMAND=(8 "$docker_cmd" "exec" "-u" "root" "$HNN_CONTAINER_NAME" "bash" "-c" "rm -rf $__hnn_out/*") + fi + convert_COMMAND_to_escaped_array + output_run_command_arguments "${ESCAPED_COMMAND[@]}" &> /dev/null + fail_on_bad_exit $? + fi + + __hnn_out_contents="$__hnn_out/." + # exceptions for windows containers + if [[ "$CONTAINER_TYPE" == "windows" ]]; then + COMMAND=(3 "cygpath.exe" "-w" "$__hnn_out_contents") + convert_COMMAND_to_escaped_array + __hnn_out_contents=$(output_run_command_arguments "${ESCAPED_COMMAND[@]}") + if [[ $? -ne 0 ]]; then + cleanup 2 + fi + + # remove leading /c/ + __hnn_out="$(echo $__hnn_out | sed 's/^\/[cC]\(.*\)$/\1/')" + stop_container_silent + fi + print_header_message_short "Copying hnn_out into container... " + COMMAND=(4 "$docker_cmd" "cp" "$__hnn_out_contents" "$HNN_CONTAINER_NAME:$__hnn_out") + convert_COMMAND_to_escaped_array + MSYS_NO_PATHCONV=1 output_run_command_arguments "${ESCAPED_COMMAND[@]}" &> /dev/null + fail_on_bad_exit $? + if [[ "$CONTAINER_TYPE" == "windows" ]]; then + start_container_silent fi } -function copy_ssh_files_to_running_container_fail { +function change_hnn_source_perms_fail { + check_var docker_cmd + check_var HNN_CONTAINER_NAME + check_var CONTAINER_TYPE + check_var USE_SSH + check_var UID + + local __container_user + local __user + + if [[ $USE_SSH -eq 0 ]]; then + __user="$UID" + else + __user="hnn_user" + fi + + if [[ "$CONTAINER_TYPE" == "windows" ]]; then + return 0 + else + __container_user="root" + __source_code_path="/home/hnn_user/hnn_source_code" + fi + print_header_message "Changing hnn_source_code permissions in container... " + COMMAND=(8 "$docker_cmd" "exec" "-u" "$__container_user" "$HNN_CONTAINER_NAME" "bash" "-c" \ + "chown -R $__user $__source_code_path") + convert_COMMAND_to_escaped_array + output_run_command_arguments "${ESCAPED_COMMAND[@]}" &> /dev/null + fail_on_bad_exit $? +} + +function copy_ssh_files_to_container_fail { # no arguments + # *** This probably won't work with windows containers because paths are in UNIX format ***** check_var docker_cmd check_var SSH_AUTHKEYS check_var SSH_PUBKEY + check_var CONTAINER_TYPE + + local __authkeys + local __pubkey + __authkeys="$SSH_AUTHKEYS" + __pubkey="$SSH_PUBKEY" + + if [[ "$CONTAINER_TYPE" == "windows" ]]; then + COMMAND=(3 "cygpath.exe" "-w" "$__authkeys") + convert_COMMAND_to_escaped_array + __authkeys=$(output_run_command_arguments "${ESCAPED_COMMAND[@]}") + if [[ $? -ne 0 ]]; then + cleanup 2 + fi + + COMMAND=(3 "cygpath.exe" "-w" "$__pubkey") + convert_COMMAND_to_escaped_array + __pubkey=$(output_run_command_arguments "${ESCAPED_COMMAND[@]}") + if [[ $? -ne 0 ]]; then + cleanup 2 + fi - if [[ $TRAVIS_TESTING -eq 1 ]] && ([[ "$OS" =~ "windows" ]] || [[ "$OS" == "wsl" ]]); then stop_container_silent fi + print_header_message_short "Copying authorized_keys file into container... " - COMMAND=(4 "$docker_cmd" "cp" "$SSH_AUTHKEYS" "$HNN_CONTAINER_NAME:/home/hnn_user/.ssh/authorized_keys") + COMMAND=(4 "$docker_cmd" "cp" "$__authkeys" "$HNN_CONTAINER_NAME:/home/hnn_user/.ssh/authorized_keys") convert_COMMAND_to_escaped_array - output_run_command_arguments "${ESCAPED_COMMAND[@]}" &> /dev/null + MSYS_NO_PATHCONV=1 output_run_command_arguments "${ESCAPED_COMMAND[@]}" &> /dev/null fail_on_bad_exit $? print_header_message_short "Copying known_hosts file into container... " - COMMAND=(4 "$docker_cmd" "cp" "$SSH_PUBKEY" "$HNN_CONTAINER_NAME:/home/hnn_user/.ssh/known_hosts") + COMMAND=(4 "$docker_cmd" "cp" "$__pubkey" "$HNN_CONTAINER_NAME:/home/hnn_user/.ssh/known_hosts") convert_COMMAND_to_escaped_array - output_run_command_arguments "${ESCAPED_COMMAND[@]}" &> /dev/null + MSYS_NO_PATHCONV=1 output_run_command_arguments "${ESCAPED_COMMAND[@]}" &> /dev/null fail_on_bad_exit $? - if [[ $TRAVIS_TESTING -eq 1 ]] && ([[ "$OS" =~ "windows" ]] || [[ "$OS" == "wsl" ]]); then + if [[ "$CONTAINER_TYPE" == "windows" ]]; then start_container_silent + # don't bother updating unix permissions + return 0 fi - print_header_message_short "Updating permissions on ssh files in container... " + print_header_message_short "Updating ownership on ssh files in container... " COMMAND=(8 "$docker_cmd" "exec" "-u" "root" "$HNN_CONTAINER_NAME" "bash" "-c" "chown -R hnn_user /home/hnn_user/.ssh") convert_COMMAND_to_escaped_array output_run_command_arguments "${ESCAPED_COMMAND[@]}" &> /dev/null fail_on_bad_exit $? + + print_header_message_short "Updating permissions on ssh authorized_keys file in container... " + COMMAND=(8 "$docker_cmd" "exec" "-u" "root" "$HNN_CONTAINER_NAME" "bash" "-c" "chmod 600 /home/hnn_user/.ssh/authorized_keys") + convert_COMMAND_to_escaped_array + output_run_command_arguments "${ESCAPED_COMMAND[@]}" &> /dev/null + fail_on_bad_exit $? } function kill_xquartz { @@ -1144,6 +1417,7 @@ function get_xquartz_port { let __timeout=30 while [[ $__timeout -gt 0 ]]; do + log_header_message "Getting running Xquartz processes... " __command="pgrep Xquartz" __pid=$(output_run_command "$__command") if [[ $? -eq 0 ]]; then @@ -1155,10 +1429,12 @@ function get_xquartz_port { __xquartz_display=$(ps $__pid|grep $__pid|sed 's/.*\(\:[0-9]\{1,\}\).*/\1/') __display_int=$(echo $__xquartz_display|sed 's/\:\([0-9]\{1,\}\)/\1/') if [[ -e "/tmp/.X11-unix/X${__display_int}" ]]; then + echo "done" >> "$LOGFILE" echo "Started XQuartz on DISPLAY $__xquartz_display" >> "$LOGFILE" break fi fi + echo "done" >> "$LOGFILE" sleep 1 (( __timeout-- )) done @@ -1189,7 +1465,7 @@ function start_xquartz { re='^[0-9]+$' __port=$(get_xquartz_port) if [[ $? -ne 0 ]]; then - exit $? + exit 2 elif ! [[ $__port =~ $re ]] ; then echo "bad xquartz port number \"$__port\"" >> "$LOGFILE" false @@ -1247,13 +1523,14 @@ function docker_pull { check_var OS check_var HNN_DOCKER_IMAGE check_var TRAVIS_TESTING + check_var CONTAINER_TYPE local __command local __docker_container local __last_used_image local __docker_image - if [[ "$OS" == "windows" ]] || [[ "$OS" == "wsl" ]]; then + if [[ "$CONTAINER_TYPE" == "windows" ]]; then __docker_image="${HNN_DOCKER_IMAGE}:win64" else __docker_image="${HNN_DOCKER_IMAGE}" @@ -1314,15 +1591,15 @@ function retry_docker_pull { fi } -function docker_run_print { +function docker_run_fail { # no arguments # will run the appropriate docker command # to leave a running containers check_var TRAVIS_TESTING check_var LOGFILE - check_var RETRY check_var OS check_var USE_SSH + check_var CONTAINER_TYPE local __timeout local __started @@ -1332,24 +1609,15 @@ function docker_run_print { local __arg local __send_args local __docker_image + local __home + local __hnn_out + local __container_xauthority __timeout=20 - if [[ $TRAVIS_TESTING -eq 1 ]]; then - find_existing_container_print - if [[ $? -eq "0" ]]; then - print_header_message "Removing old container... " - remove_container_fail - fi - elif [[ $RETRY -eq 1 ]]; then - echo "Removing old container might resolve failure. Verification required." | tee -a "$LOGFILE" - prompt_remove_container_fail - echo -n "(retry) " | tee -a "$LOGFILE" - RETRY=0 - fi - print_header_message "Starting HNN container... " + print_header_message "Creating HNN container... " - if [[ "$OS" =~ "windows" ]] || [[ "$OS" == "wsl" ]]; then + if [[ "$CONTAINER_TYPE" == "windows" ]]; then # Interesting that HOME still gets turned into a windows path. Easy to avoid # by passing the HOME variable under a new name HOME_UNIX export HOME_UNIX="${HOME}" @@ -1358,67 +1626,52 @@ function docker_run_print { __docker_image="${HNN_DOCKER_IMAGE}" fi - if [[ $TRAVIS_TESTING -eq 1 ]] && [[ "$OS" =~ "linux" ]]; then - # Importantly, the volume containing the current directory is bind mounted into - # the container and seen as /home/hnn_user/hnn_source_code, which enables tests - # inside the container to use code which has not yet been merged and been built - # as part of a container on Docker Hub. - COMMAND=(20 "${docker_cmd}" "run" "-d" \ - "-p" "22" \ - "-v" "/tmp/.X11-unix:/tmp/.X11-unix" \ - "-v" "$(pwd):/home/hnn_user/hnn_source_code" \ - "-v" "$HOME/hnn_out:$HOME/hnn_out" \ - "--env" "XAUTHORITY=/tmp/.Xauthority" "--env" "SYSTEM_USER_DIR=$HOME" \ - "--name" "$HNN_CONTAINER_NAME" "$__docker_image" "sleep" "infinity") - elif [[ $TRAVIS_TESTING -eq 1 ]] && [[ "$OS" =~ "windows" ]]; then - HNN_OUT=$(cygpath -w "$HOME/hnn_out") - COMMAND=(16 "${docker_cmd}" "run" "-d" \ - "-p" "22" \ - "-v" "$HNN_OUT:$HNN_OUT" \ - "--env" "XAUTHORITY=/c/windows/temp/.Xauthority" "--env" "SYSTEM_USER_DIR=$HOME" \ - "--name" "$HNN_CONTAINER_NAME" "$__docker_image" "sleep" "infinity") - elif [[ $TRAVIS_TESTING -eq 1 ]] && [[ "$OS" == "wsl" ]]; then - # we can't access the wsl directory, so don't create a bind-mounted volume - COMMAND=(14 "${docker_cmd}" "run" "-d" \ - "-p" "22" \ - "--env" "XAUTHORITY=/c/windows/temp/.Xauthority" "--env" "SYSTEM_USER_DIR=$HOME" \ - "--name" "$HNN_CONTAINER_NAME" "$__docker_image" "sleep" "infinity") - elif [[ $TRAVIS_TESTING -eq 1 ]]; then - # This binds port 22 in the container (for sshd) to 5000 on the host, which on docker toolbox - # is a VM. We can then configure the VM with docker-machine to allow connections to port 5000 from - # the base OS. Also connect port 6000 to allow for xforwarding of the xserver. - COMMAND=(18 "${docker_cmd}" "run" "-d" \ - "-p" "6000:6000" "-p" "5000:22" \ - "-v" "$HOME/hnn_out:$HOME/hnn_out" \ - "--env" "XAUTHORITY=/tmp/.Xauthority" "--env" "SYSTEM_USER_DIR=$HOME" \ - "--name" "$HNN_CONTAINER_NAME" "$__docker_image" "sleep" "infinity") - elif [[ "$OS" =~ "linux" ]]; then - COMMAND=(18 "${docker_cmd}" "run" "-d" \ + # exceptions for windows containers + __hnn_out="$SYSTEM_USER_DIR/hnn_out" + if [[ "$CONTAINER_TYPE" == "windows" ]]; then + COMMAND=(3 "cygpath.exe" "-w" "$__hnn_out") + convert_COMMAND_to_escaped_array + __hnn_out=$(output_run_command_arguments "${ESCAPED_COMMAND[@]}") + if [[ $? -ne 0 ]]; then + cleanup 2 + fi + fi + + __container_xauthority="/tmp/.Xauthority" + if [[ "$CONTAINER_TYPE" == "windows" ]]; then + __container_xauthority="/c/home/hnn_user/.Xauthority" + fi + + if [[ "$OS" == "mac" ]] && [[ $TRAVIS_TESTING -eq 1 ]]; then + # Workarounds for qemu docker-machine driver that doesn't support port forwarding and volume + # mounts properly. + # This binds port 22 in the container (for sshd) to 5000 on the host, which on docker toolbox + # is a VM. We can then configure the VM with docker-machine to allow connections to port 5000 from + # the base OS. Also connect port 6000 to allow for xforwarding of the xserver. + COMMAND=(14 "${docker_cmd}" "run" "-d" \ + "-p" "6000:6000" "-p" "5000:22" \ + "--env" "XAUTHORITY=$__container_xauthority" "--env" "SYSTEM_USER_DIR=$SYSTEM_USER_DIR" \ + "--name" "$HNN_CONTAINER_NAME" "$__docker_image") + elif [[ "$OS" == "linux" ]]; then + COMMAND=(16 "${docker_cmd}" "run" "-d" \ "-p" "22" \ "-v" "/tmp/.X11-unix:/tmp/.X11-unix" \ - "-v" "$HOME/hnn_out:$HOME/hnn_out" \ - "--env" "XAUTHORITY=/tmp/.Xauthority" "--env" "SYSTEM_USER_DIR=$HOME" \ - "--name" "$HNN_CONTAINER_NAME" "$__docker_image" "sleep" "infinity") - elif [[ "$OS" == "wsl" ]]; then - # we can't access the wsl directory, so don't create a bind-mounted volume - COMMAND=(14 "${docker_cmd}" "run" "-d" \ - "-p" "22" \ - "--env" "XAUTHORITY=/tmp/.Xauthority" "--env" "SYSTEM_USER_DIR=$HOME" \ - "--name" "$HNN_CONTAINER_NAME" "$__docker_image" "sleep" "infinity") - else - COMMAND=(16 "${docker_cmd}" "run" "-d" \ - "-p" "22" \ - "-v" "$HOME/hnn_out:$HOME/hnn_out" \ - "--env" "XAUTHORITY=/tmp/.Xauthority" "--env" "SYSTEM_USER_DIR=$HOME" \ - "--name" "$HNN_CONTAINER_NAME" "$__docker_image" "sleep" "infinity") + "-v" "$__hnn_out:$__hnn_out" \ + "--env" "XAUTHORITY=$__container_xauthority" "--env" "SYSTEM_USER_DIR=$SYSTEM_USER_DIR" \ + "--name" "$HNN_CONTAINER_NAME" "$__docker_image") + else # windows and mac (TRAVIS_TESTING=0) + COMMAND=(14 "${docker_cmd}" "run" "-d" \ + "-p" "22" \ + "-v" "$__hnn_out:$__hnn_out" \ + "--env" "XAUTHORITY=$__container_xauthority" "--env" "SYSTEM_USER_DIR=$SYSTEM_USER_DIR" \ + "--name" "$HNN_CONTAINER_NAME" "$__docker_image") fi convert_COMMAND_to_escaped_array MSYS_NO_PATHCONV=1 output_run_command_arguments "${ESCAPED_COMMAND[@]}" &> /dev/null if [[ $? -ne 0 ]]; then echo "*failed*" | tee -a "$LOGFILE" - false - return + cleanup 2 fi # make sure "docker ps | grep HNN_CONTAINER_NAME" succeeeds @@ -1426,16 +1679,10 @@ function docker_run_print { if [[ ! "$__started" =~ "1" ]]; then echo "*failed*" | tee -a "$LOGFILE" echo "Waited for $__timeout seconds for container to start" >> "$LOGFILE" - false - return + cleanup 2 else echo "done" | tee -a "$LOGFILE" fi - - # copy ssh auth files for TRAVIS_TESTING=1 - if [[ $USE_SSH -eq 1 ]]; then - copy_ssh_files_to_running_container_fail - fi } function print_sshd_log { @@ -1470,38 +1717,36 @@ function generate_xauth_keys_fail { fail_on_bad_exit $? } -function get_xauth_keys { +function get_host_xauth_keys { check_var LOGFILE check_var xauth_cmd check_var XAUTHORITY check_var DISPLAY + check_var DEBUG local __output + local __suppress_output + local __keys local __command_status - # don't use output_run_command_arguments because we don't want to log keys - # to log file - echo -e "\n ** Command: \"${xauth_cmd}\" -f \"$XAUTHORITY\" nlist $DISPLAY" >> "$LOGFILE" - echo -n " ** Stderr: " >> "$LOGFILE" - - if [[ "$OS" =~ "windows" ]]; then - __output=$("${xauth_cmd}" -f "$XAUTHORITY" nlist $DISPLAY 2>> "$LOGFILE") + if [[ $DEBUG -eq 1 ]]; then + __suppress_output=0 else - __output=$("${xauth_cmd}" -f "$XAUTHORITY" nlist $DISPLAY 2>> "$LOGFILE") + # don't display the keys in the log file + __suppress_output=1 fi - __command_status=$? - # don't print keys in log - if [ -n "$__output" ]; then - echo " ** Stdout: ** suppresed **" >> "$LOGFILE" - else - echo " ** Stdout: " >> "$LOGFILE" - fi - echo "$__output" - if [[ $__command_status -eq 0 ]]; then - true + log_header_message "Retrieving host xauth keys... " + COMMAND=(6 "${xauth_cmd}" "-f" "$XAUTHORITY" "-ni" "nlist" "$DISPLAY") + convert_COMMAND_to_escaped_array + __keys=$(SUPPRESS_OUTPUT=$__suppress_output output_run_command_arguments "${ESCAPED_COMMAND[@]}") + __command_status=$? + if [[ $__command_status -ne 0 ]]; then + echo "*failed*" >> "$LOGFILE" + exit $__command_status else - false + echo "done" >> "$LOGFILE" + echo "$__keys" fi } @@ -1513,13 +1758,13 @@ function check_xauth_keys_print { local __command_status print_header_message "Checking for X11 authentication keys... " - __output=$(get_xauth_keys) + __output=$(get_host_xauth_keys) __command_status=$? + echo >> "$LOGFILE" if [[ $__command_status -ne 0 ]]; then echo "*failed*" | tee -a "$LOGFILE" exit $__command_status - fi - if [[ -z "$__output" ]]; then + elif [[ -z "$__output" ]]; then echo "no valid keys" | tee -a "$LOGFILE" false else @@ -1590,6 +1835,8 @@ function get_display_for_gui { # no arguments # set DISPLAY environment variable for the appropriate OS and/or testing environment check_var DOCKER_TOOLBOX + check_var CONTAINER_TYPE + check_var TRAVIS_TESTING local __display local __port @@ -1601,23 +1848,37 @@ function get_display_for_gui { fi # set DISPLAY for GUI - if [[ "$OS" =~ "linux" ]]; then - # linux can use direct port - if [[ $TRAVIS_TESTING -eq 1 ]]; then - [[ $DISPLAY ]] && __display=$DISPLAY || __display=":$__port" + if [[ "$CONTAINER_TYPE" == "windows" ]]; then + log_header_message "Getting IP for DISPLAY from container... " + COMMAND=(6 "$docker_cmd" "exec" "$HNN_CONTAINER_NAME" "bash" "-c" "ipconfig.exe | grep 'Default Gateway'| sed -e 's/^.*:[ \t]*\(.*\)/\1/'") + convert_COMMAND_to_escaped_array + __ip=$(output_run_command_arguments "${ESCAPED_COMMAND[@]}") + if [[ $? -ne 0 ]]; then + echo "*failed*" >> "$LOGFILE" + exit 2 else - __display=":$__port" + echo "done" >> "$LOGFILE" fi + __display="$__ip:$__port" else - if [[ $TRAVIS_TESTING -eq 1 ]]; then - # qemu driver - __display="10.0.2.2:$__port" - elif [[ $DOCKER_TOOLBOX -eq 1 ]]; then - # virtualbox driver - __display="192.168.99.1:$__port" + if [[ "$OS" =~ "linux" ]]; then + # linux can use direct port + if [[ $TRAVIS_TESTING -eq 1 ]]; then + [[ $DISPLAY ]] && __display=$DISPLAY || __display=":$__port" + else + __display=":$__port" + fi else - # docker desktop - __display="host.docker.internal:$__port" + if [[ $TRAVIS_TESTING -eq 1 ]]; then + # qemu driver + __display="10.0.2.2:$__port" + elif [[ $DOCKER_TOOLBOX -eq 1 ]]; then + # virtualbox driver + __display="192.168.99.1:$__port" + else + # docker desktop + __display="host.docker.internal:$__port" + fi fi fi @@ -1625,21 +1886,37 @@ function get_display_for_gui { } -function get_host_xauth_key { +function get_valid_host_xauth_key { # no arguments # retrieves xauth key from host - check_var LOGFILE check_var DISPLAY check_var xauth_cmd + check_var XAUTHORITY + check_var DEBUG + local __keys local __key + local __command_status - echo -e "\n ** Command: xauth -ni nlist $DISPLAY | grep '^fff' | head -1 | awk '{print \$NF}'" >> "$LOGFILE" - __key=$("${xauth_cmd}" -ni nlist $DISPLAY | grep '^fff' | head -1 | awk '{print $NF}' 2>> "$LOGFILE") + __keys=$(get_host_xauth_keys) + __command_status=$? if [[ $? -ne 0 ]]; then - return 1 + return $__command_status else - echo $__key + log_header_message "Validating host xauth keys... " + __key="$(echo $__keys | grep 'ffff' | head -1 | awk '{print $NF}')" + if [[ -z "$__key" ]]; then + # no output means no valid keys + echo "*failed*" >> "$LOGFILE" + echo -e " ** Error: no valid key **" >> "$LOGFILE" + return 1 + else + echo "done" >> "$LOGFILE" + if [[ $DEBUG -eq 1 ]]; then + echo "Valid key: =START=$__key=END=" >> "$LOGFILE" + fi + echo "$__key" + fi fi } @@ -1649,26 +1926,54 @@ function get_container_xauth_key { check_args "$@" $# 1 - check_var LOGFILE check_var docker_cmd check_var HNN_CONTAINER_NAME + check_var DEBUG + local __keys local __key local __user local __display + local __suppress_output + local __command_status __user="$1" __display=$(get_display_for_gui) if [[ $? -ne 0 ]]; then - cleanup $? + cleanup 2 fi - echo -e "\n ** Command: \"$docker_cmd\" exec -u \"$__user\" \"$HNN_CONTAINER_NAME\" bash -c \"timeout 5 xauth -ni nlist \"$__display\" | grep '^ffff' | head -1 | awk '{print \$NF}'\"" >> "$LOGFILE" - echo -n " ** Stderr: " >> "$LOGFILE" - __key=$("$docker_cmd" exec -u "$__user" "$HNN_CONTAINER_NAME" bash -c "timeout 5 xauth -ni nlist "$__display" | grep '^ffff' | head -1 | awk '{print \$NF}'" 2>> "$LOGFILE") - if [[ $? -ne 0 ]]; then - return $? + log_header_message "Retrieving container xauth keys... " + COMMAND=(8 "$docker_cmd" exec "-u" "$__user" "$HNN_CONTAINER_NAME" "bash" "-c" "timeout 5 xauth -ni nlist $__display") + convert_COMMAND_to_escaped_array + # don't display the keys in the log file + if [[ $DEBUG -eq 1 ]]; then + __suppress_output=0 else - echo $__key + __suppress_output=1 + fi + __keys=$(SUPPRESS_OUTPUT=$__suppress_output output_run_command_arguments "${ESCAPED_COMMAND[@]}") + __command_status=$? + if [[ $__command_status -ne 0 ]]; then + echo "*failed*" >> "$LOGFILE" + exit $__command_status + else + echo "done" >> "$LOGFILE" + + log_header_message "Validating container xauth keys... " + __key="$(echo $__keys | grep 'ffff' | head -1 | awk '{print $NF}')" + if [[ -z "$__key" ]]; then + # no output means no valid keys + echo "*failed*" >> "$LOGFILE" + echo -e " ** Error: no valid key **" >> "$LOGFILE" + return 1 + else + echo "done" >> "$LOGFILE" + fi + + if [[ $DEBUG -eq 1 ]]; then + echo "Valid key: =START=$__key=END=" >> "$LOGFILE" + fi + echo "$__key" fi } @@ -1678,44 +1983,42 @@ function check_container_xauth_print { check_var LOGFILE check_var RETRY + check_var CONTAINER_TYPE local __user + local __command_status local __host_key - __user=$1 - __host_key=$(get_host_xauth_key) - if [[ $? -ne 0 ]]; then - exit $? - elif [[ -z $__host_key ]]; then - echo $__host_key - # no output means no valid keys - echo -e " ** Error: no valid key **" >> "$LOGFILE" - return 1 - fi + local __key - if [[ $TRAVIS_TESTING -eq 1 ]] && ([[ "$OS" =~ "windows" ]] || [[ "$OS" == "wsl" ]]); then - # always return success because we can't run xauth within container - return 0 + __user=$1 + __host_key=$(get_valid_host_xauth_key) + __command_status=$? + if [[ $__command_status -ne 0 ]]; then + cleanup 2 fi if [[ $RETRY -eq 1 ]]; then echo -n "(retry) " | tee -a "$LOGFILE" RETRY=0 fi - print_header_message "Checking that xauth key in container matches... " - __key=$(get_container_xauth_key $1) + + if [[ "$CONTAINER_TYPE" == "windows" ]]; then + __key=$(get_container_xauth_key "hnn_user") + else + __key=$(get_container_xauth_key $1) + fi __command_status=$? + + + print_header_message "Checking that xauth key in container matches... " if [[ $__command_status -ne 0 ]]; then echo "*failed*" | tee -a "$LOGFILE" echo -e " ** Error: could not retrieve keys from container **" >> "$LOGFILE" - exit $__command_status - elif [[ -z $__key ]]; then - # no output means no valid keys - echo "*failed*" | tee -a "$LOGFILE" - echo -e " ** Error: no valid key **" >> "$LOGFILE" return 1 elif [[ "$__host_key" == "$__key" ]]; then echo "done" | tee -a "$LOGFILE" else + echo "*$__host_key* != *$__key*" >> "$LOGFILE" echo "*failed*" | tee -a "$LOGFILE" echo -e " ** Error: key mismatch **" >> "$LOGFILE" return 1 @@ -1758,7 +2061,6 @@ function setup_xauthority_in_container_fail { # make sure that xauth works in container now check_container_xauth_print "$__user" if [[ $? -ne 0 ]]; then - echo "*failed*" | tee -a "$LOGFILE" cleanup 2 fi fi @@ -1799,13 +2101,13 @@ function check_xquartz_listening { [[ $OS ]] || export OS=$(get_os) if [[ $? -ne 0 ]]; then - cleanup $? + cleanup 2 fi # before assuming port 0, check if xquartz is already running __current_port=$(get_xquartz_port) if [[ $? -ne 0 ]]; then - exit $? + exit 2 elif [[ -z $__current_port ]]; then __current_port="0" fi @@ -1819,7 +2121,7 @@ function check_xquartz_listening { fi __current_port=$(restart_xquartz_fail) if [[ $? -ne 0 ]]; then - exit $? + exit 2 fi sleep 1 (( __retry++ )) @@ -1927,18 +2229,18 @@ function generate_ssh_auth_keys_fail { cleanup 2 fi - echo -e "\n ** Command: { echo -n 'command=\"/home/hnn_user/start_hnn.sh\" '; cat \"$SSH_PUBKEY\"; } > \"$SSH_AUTHKEYS\"" >> "$LOGFILE" - echo -n " ** Stderr: " >> "$LOGFILE" - { echo -n 'command="/home/hnn_user/start_hnn.sh" '; cat "$SSH_PUBKEY"; } > "$SSH_AUTHKEYS" 2>> "$LOGFILE" + COMMAND=(3 "chmod" "600" "$SSH_PRIVKEY") + convert_COMMAND_to_escaped_array + output_run_command_arguments "${ESCAPED_COMMAND[@]}" &> /dev/null if [[ $? -ne "0" ]]; then echo "*failed*" | tee -a "$LOGFILE" cleanup 2 fi echo >> "$LOGFILE" - COMMAND=(3 "chmod" "600" "$SSH_AUTHKEYS") - convert_COMMAND_to_escaped_array - output_run_command_arguments "${ESCAPED_COMMAND[@]}" &> /dev/null + echo -e "\n ** Command: { echo -n 'command=\"/home/hnn_user/start_hnn.sh\" '; cat \"$SSH_PUBKEY\"; } > \"$SSH_AUTHKEYS\"" >> "$LOGFILE" + echo -n " ** Stderr: " >> "$LOGFILE" + { echo -n 'command="/home/hnn_user/start_hnn.sh" '; cat "$SSH_PUBKEY"; } > "$SSH_AUTHKEYS" 2>> "$LOGFILE" if [[ $? -ne "0" ]]; then echo "*failed*" | tee -a "$LOGFILE" cleanup 2 @@ -2007,10 +2309,13 @@ function check_docker_working_print { COMMAND=(4 "timeout" "5" "$docker_cmd" version) convert_COMMAND_to_escaped_array __output=$(output_run_command_arguments "${ESCAPED_COMMAND[@]}") - if [[ ! -z $__output ]] && [[ $? -ne "0" ]]; then + if [[ $? -ne 0 ]]; then echo "*failed*" | tee -a "$LOGFILE" false - elif [[ $? -eq "124" ]]; then + elif [[ -z "$__output" ]]; then + echo "*failed*" | tee -a "$LOGFILE" + false + elif [[ $? -eq 124 ]]; then echo "*failed*" | tee -a "$LOGFILE" echo "Error: timed out connecting to Docker. Please check Docker install" | tee -a "$LOGFILE" false @@ -2019,6 +2324,38 @@ function check_docker_working_print { fi } +function get_docker_container_type { + check_var LOGFILE + check_var docker_cmd + + local __output + + print_header_message "Checking Docker container type... " + COMMAND=(4 "timeout" "5" "$docker_cmd" info) + convert_COMMAND_to_escaped_array + __output=$(output_run_command_arguments "${ESCAPED_COMMAND[@]}") + if [[ $? -ne 0 ]]; then + echo "*failed*" | tee -a "$LOGFILE" + cleanup 2 + elif [[ -z "$__output" ]]; then + echo "*failed*" | tee -a "$LOGFILE" + cleanup 2 + elif [[ $? -eq 124 ]]; then + echo "*failed*" | tee -a "$LOGFILE" + echo "Error: timed out connecting to Docker. Please check Docker install" | tee -a "$LOGFILE" + cleanup 2 + else + if [[ "$(echo $__output | grep "Storage Driver:")" =~ "windowsfilter" ]]; then + CONTAINER_TYPE="windows" + else + CONTAINER_TYPE="linux" + fi + echo "$CONTAINER_TYPE" | tee -a "$LOGFILE" + fi + + export CONTAINER_TYPE +} + function remove_hnn_image_fail { check_var docker_cmd check_var HNN_DOCKER_IMAGE @@ -2045,13 +2382,29 @@ function remove_hnn_image_fail { function check_vcxsrv_running_print { check_var LOGFILE + check_var TRAVIS_TESTING print_header_message_short "Checking if VcXsrv is running... " - export VCXSRV_PID=$(tasklist | grep vcxsrv | awk '{print $2}' 2> /dev/null) + if [[ "$OS" == "wsl" ]]; then + TASKS=$(powershell.exe -c '& tasklist') + if [[ $? -ne 0 ]]; then + echo "*failed*" | tee -a "$LOGFILE" + cleanup 2 + fi + export VCXSRV_PID=$(echo "$TASKS"| grep vcxsrv | awk '{print $2}' 2> /dev/null) + else + export VCXSRV_PID=$(tasklist | grep vcxsrv | awk '{print $2}' 2> /dev/null) + fi + if [ -n "${VCXSRV_PID}" ]; then echo -e "\nVcXsrv running with PID $VCXSRV_PID" >> "$LOGFILE" echo "yes" | tee -a "$LOGFILE" - echo "WARNING: continuing with existing VcXsrv process. You many need to stop it and let this script start VcXsrv for the GUI to display" + if [[ $TRAVIS_TESTING -eq 1 ]]; then + echo -e "\nError: Travis expects that VcXsrv will be started every time" + cleanup 1 + else + echo "WARNING: continuing with existing VcXsrv process. You many need to stop it and let this script start VcXsrv for the GUI to display" + fi else echo "no" | tee -a "$LOGFILE" export VCXSRV_PID= @@ -2078,7 +2431,7 @@ function find_command_suggested_path { __program="$1" __path="$2" - if [[ "$OS" =~ "windows" ]]; then + if [[ "$OS" == "windows" ]] || [[ "$OS" == "wsl" ]]; then __executable_name="$__program.exe" else __executable_name="$__program" From bf7046cdbd492e00da685e324e38431ba585ae0e Mon Sep 17 00:00:00 2001 From: Blake Caldwell Date: Thu, 14 May 2020 17:31:49 -0400 Subject: [PATCH 3/8] MAINT: ubuntu: update installer for WSL WSL is now installed and tested by Travis. Update the installer.sh script to work with WSL as used by Travis. Also interleave concurrent install and download commands to speed up the script's completion. --- installer/ubuntu/installer.sh | 214 ++++++++++++++++++++++++---------- 1 file changed, 153 insertions(+), 61 deletions(-) diff --git a/installer/ubuntu/installer.sh b/installer/ubuntu/installer.sh index 1432c6200..65e7dc8ce 100755 --- a/installer/ubuntu/installer.sh +++ b/installer/ubuntu/installer.sh @@ -5,93 +5,150 @@ set -e [[ ${NEURON_VERSION} ]] || NEURON_VERSION=7.7 [[ "$LOGFILE" ]] || LOGFILE="ubuntu_install.log" +function start_download { + echo "Downloading $2" + let __retries=5 + while [[ $__retries -gt 0 ]]; do + curl -Lo "$1" --retry 5 --retry-delay 30 "$2" && break + (( __retries-- )) + done + if [[ $__retries -eq 0 ]]; then + echo "Error: failed to download $2." + exit 1 + fi +} + +function wait_for_pid { + echo -n "Waiting for PID $1... " + wait $1 && { + echo "done" + echo "Finished $2" + } || { + echo "*failed*" + echo "Error: failed $2" + exit 1 + } +} + +function script_fail { + echo -ne "\n******* install script failed. output from log below *******\n" + cat "$LOGFILE" + exit 2 +} + +trap script_fail EXIT + DISTRIB=$(grep DISTRIB_CODENAME /etc/lsb-release | cut -d'=' -f2) echo "Starting installation of HNN on dectected Ubuntu OS: $DISTRIB" | tee -a "$LOGFILE" -echo "Output in log file: $LOGFILE" | tee -a "$LOGFILE" - -if [[ "$DISTRIB" =~ "xenial" ]]; then - PYTHON_VERSION=3.5 -elif [[ "$DISTRIB" =~ "bionic" ]]; then - PYTHON_VERSION=3.6 -elif [[ "$DISTRIB" =~ "disco" ]]; then - PYTHON_VERSION=3.7 -else - echo "Error: Ubuntu distribtion $DISTRIB not supported" | tee -a "$LOGFILE" - exit 1 -fi -echo "Will install python version: $PYTHON_VERSION" | tee -a "$LOGFILE" - +echo "Output in log file: $LOGFILE" + +[[ $PYTHON_VERSION ]] || { + if [[ "$DISTRIB" =~ "xenial" ]]; then + PYTHON_VERSION=3.5 + elif [[ "$DISTRIB" =~ "bionic" ]]; then + PYTHON_VERSION=3.6 + elif [[ "$DISTRIB" =~ "disco" ]]; then + PYTHON_VERSION=3.7 + else + echo "Error: Ubuntu distribtion $DISTRIB not supported" | tee -a "$LOGFILE" + exit 1 + fi +} +echo "Using python version $PYTHON_VERSION" | tee -a "$LOGFILE" # avoid questions from debconf export DEBIAN_FRONTEND=noninteractive +echo "Updating package repository..." | tee -a "$LOGFILE" +sudo -E apt-get update &> "$LOGFILE" + +echo "Downloading Python packages..." | tee -a "$LOGFILE" +apt-get download \ + make gcc g++ python3 python3-pip python3-dev python3-tk python3-setuptools \ + libncurses5 libreadline5 libdbus-1-3 libopenmpi-dev &> "$LOGFILE" & +APT_DOWNLOAD=$! + +echo "Waiting for package download to finish..." +NAME="downloading python packages" +wait_for_pid "${APT_DOWNLOAD}" "$NAME" + +echo "Downloading NEURON..." +URL="https://neuron.yale.edu/ftp/neuron/versions/v${NEURON_VERSION}/nrn-${NEURON_VERSION}.$(uname -p)-linux.deb" +FILENAME="/tmp/nrn.deb" +start_download "$FILENAME" "$URL" > /dev/null & +NEURON_DOWNLOAD=$! + echo "Updating OS python packages..." | tee -a "$LOGFILE" -sudo apt-get update &> "$LOGFILE" if [[ "${PYTHON_VERSION}" =~ "3.7" ]] && [[ "$DISTRIB" =~ "bionic" ]]; then - sudo apt-get install --no-install-recommends -y python3.7 python3-pip python3.7-tk python3.7-dev &> "$LOGFILE" && \ + sudo -E apt-get install --no-install-recommends -y python3.7 python3-pip python3.7-tk python3.7-dev &> "$LOGFILE" && \ sudo python3.7 -m pip install --upgrade pip setuptools &> "$LOGFILE" + sudo ln -s /usr/lib/python3/dist-packages/apt_pkg.cpython-36m-x86_64-linux-gnu.so \ + /usr/lib/python3/dist-packages/apt_pkg.so else - sudo apt-get install --no-install-recommends -y python3 python3-pip python3-tk python3-setuptools &> "$LOGFILE" && \ + sudo -E apt-get install --no-install-recommends -y python3 python3-pip python3-tk python3-setuptools &> "$LOGFILE" && \ sudo pip3 install --upgrade pip &> "$LOGFILE" fi -echo "Installing OS compilation toolchain..." | tee -a "$LOGFILE" -# get prerequisites from pip. requires gcc to build psutil -sudo apt-get install --no-install-recommends -y \ - make gcc g++ python3-dev &> "$LOGFILE" - -which python > /dev/null 2>&1 -if [[ $? -eq 0 ]]; then - # there is another python executable that might be version 2.7 +if which python3 &> /dev/null; then export PYTHON=$(which python3) - PIP=pip3 -else + if pip3 -V &> /dev/null; then + PIP=pip3 + else + PIP=pip + fi +elif which python &> /dev/null; then + export PYTHON=$(which python) PIP=pip fi echo "Using python: $PYTHON" | tee -a "$LOGFILE" -echo "Installing python pacakges for HNN with pip..." | tee -a "$LOGFILE" +echo "Downloading python packages for HNN with pip..." | tee -a "$LOGFILE" +$PIP download matplotlib PyOpenGL \ + pyqt5 pyqtgraph scipy numpy nlopt psutil &> "$LOGFILE" & +PIP_PID=$! + +echo "Installing OS compilation toolchain..." | tee -a "$LOGFILE" +# get prerequisites from pip. requires gcc to build psutil +sudo -E apt-get install --no-install-recommends -y \ + make gcc g++ python3-dev &> "$LOGFILE" + +echo "Waiting for python packages for HNN downloads to finish..." +NAME="downloading python packages for HNN " +wait_for_pid "${PIP_PID}" "$NAME" + +echo "Installing python packages for HNN with pip..." | tee -a "$LOGFILE" $PIP install --no-cache-dir --user matplotlib PyOpenGL \ pyqt5 pyqtgraph scipy numpy nlopt psutil &> "$LOGFILE" -# base prerequisites packages -echo "Installing OS openmpi..." | tee -a "$LOGFILE" -sudo apt-get install --no-install-recommends -y \ - openmpi-bin lsof &> "$LOGFILE" - -# Qt prerequisites packages -echo "Installing Qt OS prerequisites..." | tee -a "$LOGFILE" -sudo apt-get install --no-install-recommends -y \ - libfontconfig libxext6 libx11-xcb1 libxcb-glx0 \ - libxkbcommon-x11-0 &> "$LOGFILE" +echo "Waiting for NEURON download to finish..." +NAME="downloading NEURON package" +wait_for_pid "${NEURON_DOWNLOAD}" "$NAME" -echo "Installing NEURON prerequisites..." | tee -a "$LOGFILE" -# NEURON runtime prerequisites -sudo apt-get install --no-install-recommends -y \ - libncurses5 libreadline5 libdbus-1-3 libopenmpi-dev &> "$LOGFILE" +echo "Downloading runtime prerequisite packages..." | tee -a "$LOGFILE" +apt-get download \ + openmpi-bin lsof libfontconfig1 libxext6 libx11-xcb1 libxcb-glx0 \ + libxkbcommon-x11-0 libgl1-mesa-glx \ + libc6-dev libtinfo-dev libncurses5-dev \ + libx11-dev libreadline-dev &> "$LOGFILE" & +APT_DOWNLOAD=$! # Install NEURON echo "Installing NEURON $NEURON_VERSION precompiled package..." | tee -a "$LOGFILE" -wget -q https://neuron.yale.edu/ftp/neuron/versions/v${NEURON_VERSION}/nrn-${NEURON_VERSION}.$(uname -p)-linux.deb -O /tmp/nrn.deb &> "$LOGFILE" && \ - sudo dpkg -i /tmp/nrn.deb &> "$LOGFILE" && \ - rm -f /tmp/nrn.deb &> "$LOGFILE" - -# HNN build prerequisites -echo "Installing HNN build prerequisites..." | tee -a "$LOGFILE" -sudo apt-get install --no-install-recommends -y \ - libc6-dev libtinfo-dev libncurses-dev \ - libx11-dev libreadline-dev &> "$LOGFILE" +(sudo -E dpkg -i /tmp/nrn.deb &> "$LOGFILE" && \ + rm -f /tmp/nrn.deb &> "$LOGFILE" ) & +NEURON_PID=$! # save dir installing hnn to startdir=$(pwd) if [[ $TRAVIS_TESTING -ne 1 ]]; then # setup HNN itself - if [ -d "$startdir/hnn_source_code" ]; then + source_code_dir="$startdir/hnn_source_code" + if [ -d "$source_code_dir" ]; then echo "Updating HNN source code..." | tee -a "$LOGFILE" cd hnn_source_code - if [ -d "$startdir/hnn_source_code/.git" ]; then + if [ -d "$source_code_dir/.git" ]; then git pull origin master &> "$LOGFILE" fi else @@ -99,20 +156,30 @@ if [[ $TRAVIS_TESTING -ne 1 ]]; then git clone https://github.com/jonescompneurolab/hnn.git hnn_source_code &> "$LOGFILE" && \ cd hnn_source_code fi +else + source_code_dir="$startdir" + if [[ ! -f hnn.py ]]; then + echo "Couldn't find HNN source code at $startdir" | tee -a "$LOGFILE" + exit 1 + fi +fi +echo "Waiting for NEURON install to finish..." +NAME="installing NEURON" +wait_for_pid "${NEURON_PID}" "$NAME" + +if [[ $TRAVIS_TESTING -ne 1 ]]; then echo "Building HNN..." | tee -a "$LOGFILE" make -j4 &> "$LOGFILE" - - cd "$startdir" + MAKE_PID=$! fi -# Clean up a little -echo "Cleaning up..." | tee -a "$LOGFILE" -sudo apt-get clean &> "$LOGFILE" - # create the global session variables echo '# these lines define global session variables for HNN' >> ~/.bashrc -echo "export PATH=\$PATH:\"$startdir/hnn_source_code\"" >> ~/.bashrc +echo "export PATH=\$PATH:\"$source_code_dir\"" >> ~/.bashrc +if [[ "${PYTHON_VERSION}" =~ "3.6" ]] && [[ "${PYTHON_VERSION}" =~ "3.7" ]]; then + echo "export PYTHONPATH=/usr/local/nrn/lib/python" >> ~/.bashrc +fi if [[ -d "$HOME/Desktop" ]]; then { @@ -124,5 +191,30 @@ if [[ -d "$HOME/Desktop" ]]; then } &> "$LOGFILE" fi +echo "Waiting for prerequisite package downloads to finish..." +NAME="downloading prerequisite packages" +wait_for_pid "${APT_DOWNLOAD}" "$NAME" + +echo "Installing prerequisites..." | tee -a "$LOGFILE" +sudo -E apt-get install --no-install-recommends -y \ + openmpi-bin lsof \ + libfontconfig1 libxext6 libx11-xcb1 libxcb-glx0 \ + libxkbcommon-x11-0 libgl1-mesa-glx \ + libncurses5 libreadline5 libdbus-1-3 libopenmpi-dev \ + libc6-dev libtinfo-dev libncurses5-dev \ + libx11-dev libreadline-dev &> "$LOGFILE" + +# Clean up a little +echo "Cleaning up..." | tee -a "$LOGFILE" +sudo -E apt-get clean &> "$LOGFILE" + +if [[ $TRAVIS_TESTING -ne 1 ]]; then + echo "Waiting for HNN module build to finish..." + NAME="building HNN modules" + wait_for_pid "${MAKE_PID}" "$NAME" +fi + echo "HNN installation successful" | tee -a "$LOGFILE" -echo "Source code is at $startdir/hnn_source_code" | tee -a "$LOGFILE" +echo "Source code is at $source_code_dir" | tee -a "$LOGFILE" + +trap EXIT \ No newline at end of file From e9f5fa8941034c068b307ab65207b16bf898379c Mon Sep 17 00:00:00 2001 From: Blake Caldwell Date: Fri, 15 May 2020 07:10:26 -0400 Subject: [PATCH 4/8] TST: mac: show qemu errors and retry --- scripts/setup-travis-mac.sh | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/scripts/setup-travis-mac.sh b/scripts/setup-travis-mac.sh index dc4803c0b..f8c24fcda 100755 --- a/scripts/setup-travis-mac.sh +++ b/scripts/setup-travis-mac.sh @@ -1,12 +1,12 @@ #!/bin/bash -set -xe +set -e export TRAVIS_TESTING=1 source scripts/utils.sh export -f sha256sum -export DOCKER_IMAGE_NAME="jonescompneurolab/hnn:master" +export DOCKER_IMAGE_NAME="jonescompneurolab/hnn:latest" export BASE_QEMU_OPTS="--disable-cocoa --disable-curses --disable-vnc --disable-vde \ --disable-pie --disable-libusb --disable-hax --disable-kvm \ --disable-debug-info --disable-docs --disable-nettle \ @@ -175,17 +175,17 @@ start_download "$FILENAME" "$URL" & NRN_PID=$! if [[ $HOMEBREW_QEMU -eq 1 ]] || [[ $BUILD_QEMU -eq 1 ]]; then -echo "Waiting for qemu build to finish" -NAME="building qemu" -wait_for_pid "${MAKE_PID}" "$NAME" + echo "Waiting for qemu build to finish" + NAME="building qemu" + wait_for_pid "${MAKE_PID}" "$NAME" -make install -command cd ${TRAVIS_BUILD_DIR} + make install + command cd ${TRAVIS_BUILD_DIR} fi # create default VM with docker-machine echo "Starting qemu VM..." -docker-machine -D create --driver qemu --qemu-cache-mode unsafe --qemu-cpu-count 2 default & +docker-machine -D create --driver qemu --qemu-cache-mode unsafe --qemu-cpu-count 2 default &> machine.log & export MACHINE_PID=$! curl -Lo "$HOME/download-frozen-image-v2.sh" https://raw.githubusercontent.com/moby/moby/master/contrib/download-frozen-image-v2.sh @@ -211,7 +211,17 @@ conda install -y -n hnn -c conda-forge nlopt pip download flake8 pytest pytest-cov coverage coveralls mne echo "Waiting for VM to start..." -wait $MACHINE_PID +wait $MACHINE_PID || COMMAND_STATUS=$? +if [[ $COMMAND_STATUS -ne 0 ]]; then + echo -e "\n********** Docker machine failed with status $COMMAND_STATUS. Log below *************\n" + cat machine.log + if [ -f /tmp/qemu.log ]; then + echo -e "\n********** QEMU log *************\n" + cat /tmp/qemu.log + fi + echo "Trying to start VM again..." + docker-machine -D create --driver qemu --qemu-cache-mode unsafe --qemu-cpu-count 2 default +fi echo "VM running docker is up" # set up environment variables to use docker within VM @@ -223,9 +233,10 @@ NAME="downloading HNN docker image" wait_for_pid "${IMAGE_PID}" "$NAME" echo "Loading downloaded image into docker" (tar -cC "$HOME/docker_image" . | docker load && \ -docker tag jonescompneurolab/hnn:master jonescompneurolab/hnn:latest && \ touch $HOME/docker_image_loaded) & +# docker tag jonescompneurolab/hnn:latest jonescompneurolab/hnn:latest && \ + # hack so that NEURON install doesn't take forever sudo kill -9 $NRN_INSTALL_PID && wait $NRN_INSTALL_PID || { if [[ $? -eq 137 ]]; then @@ -235,4 +246,4 @@ else fi } -echo "Install finished" \ No newline at end of file +echo "Install finished" From 314cd0b03f60e1ddce05214a620ddeae00cf0952 Mon Sep 17 00:00:00 2001 From: Blake Caldwell Date: Thu, 14 May 2020 17:25:15 -0400 Subject: [PATCH 5/8] TST: enable WSL testing --- .travis.yml | 202 ++++++++++++++++++++------------ scripts/docker_functions.sh | 5 + scripts/run-travis-windows.sh | 2 +- scripts/setup-travis-windows.sh | 74 ++++++------ scripts/setup-wsl.ps1 | 26 ++++ scripts/start-test-as-user.ps1 | 44 ++++--- scripts/utils.sh | 5 +- 7 files changed, 227 insertions(+), 131 deletions(-) create mode 100644 scripts/setup-wsl.ps1 diff --git a/.travis.yml b/.travis.yml index b623ab2f5..44ee632bb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -89,49 +89,34 @@ matrix: - NEURON_VERSION=7.7 before_install: - - set -e - - | + - set -e # error on any command failure + - | # function exports export TRAVIS_TESTING=1 - source scripts/utils.sh - source scripts/docker_functions.sh + # source utility functions export LOGFILE="hnn_travis.log" - set_globals - - # override cleanup function in docker_functions.sh - function cleanup { - check_var LOGFILE - - local __failed - - __failed=$1 - - echo -e "\n=====================" >> $LOGFILE - echo "cleanup() called from: ${FUNCNAME[1]} (L:${BASH_LINENO[0]})" >> $LOGFILE - - if [[ $__failed -ne "0" ]]; then - echo -e "\n======================================" - echo "Error: Please see log output for more details" - cat $LOGFILE - return $__failed - fi - } - + source scripts/docker_functions.sh + source scripts/utils.sh export -f cleanup - export DOCKER_IMAGE_NAME="jonescompneurolab/hnn:master" - export BASE_QEMU_OPTS="--disable-cocoa --disable-curses --disable-vnc --disable-vde \ - --disable-pie --disable-libusb --disable-hax --disable-kvm \ - --disable-debug-info --disable-docs --disable-nettle \ - --disable-sparse --disable-guest-agent --disable-qom-cast-debug \ - --disable-lzo --disable-bzip2 --disable-fdt \ - --disable-tpm --disable-replication --disable-modules --disable-blobs \ - --disable-gnutls --disable-vhost-crypto --disable-live-block-migration \ - --disable-user --disable-vhost-net" + # some functions require $docker_cmd and $CONTAINER_TYPE + set_globals - | if [ "${TRAVIS_OS_NAME}" == "osx" ]; then # install osx prerequisites + echo "Installing macOS prerequisites" + scripts/setup-travis-mac.sh + export PATH=${HOME}/miniconda/bin:$PATH + export PATH=$PATH:/Applications/NEURON-${NEURON_VERSION}/nrn/x86_64/bin + export PYTHONPATH=/Applications/NEURON-${NEURON_VERSION}/nrn/lib/python:$PYTHONPATH + export PATH=$PATH:/Applications/Docker.app/Contents/Resources/bin + export PYTHON=python3 + eval "$(docker-machine env default)" + + # check container type + find_program_print docker && export docker_cmd || script_fail + get_docker_container_type && export CONTAINER_TYPE || script_fail # set up DISPLAY and make sure that xquartz is ready check_xquartz_listening || { @@ -141,17 +126,16 @@ before_install: # accept TCP connections in hnn_docker.sh later xset -display $DISPLAY -q > /dev/null - export PATH=${HOME}/miniconda/bin:$PATH - export PATH=$PATH:/Applications/NEURON-${NEURON_VERSION}/nrn/x86_64/bin - export PYTHONPATH=/Applications/NEURON-${NEURON_VERSION}/nrn/lib/python:$PYTHONPATH - export PATH=$PATH:/Applications/Docker.app/Contents/Resources/bin - export PYTHON=python3 - eval "$(docker-machine env default)" - source activate hnn && echo "activated conda HNN environment" fi - - | - if [ "${TRAVIS_OS_NAME}" == "windows" ]; then # install windows prerequisites + - | # windows + if [ "${TRAVIS_OS_NAME}" == "windows" ]; then + echo "Installing windows prerequisites" + + # check container type + find_program_print docker && export docker_cmd || script_fail + get_docker_container_type && export CONTAINER_TYPE || script_fail + scripts/setup-travis-windows.sh # add miniconda python to the path @@ -160,34 +144,55 @@ before_install: export PATH=$HOME/Miniconda3/envs/hnn/Scripts:$PATH export PATH=$HOME/Miniconda3/envs/hnn/Library/bin:$PATH + # for using X server + export PATH="$PATH:/c/Program\ Files/VcXsrv" + + # for MESA dll's + export PATH=$PATH:/c/tools/msys64/mingw64/bin + + # for sharing with WSL environment + export WSLENV=TRAVIS_TESTING/u + # set other variables for neuron and HNN export PATH=$PATH:/c/nrn/bin export DISPLAY="localhost:0" export NEURONHOME=/c/nrn export PYTHON=python fi - - | + - | # Linux if [ "${TRAVIS_OS_NAME}" == "linux" ]; then + echo "Installing Linux prerequisites" + + # check container type + find_program_print docker && export docker_cmd || script_fail + get_docker_container_type && export CONTAINER_TYPE || script_fail + echo "Starting fake Xserver" Xvfb :0 -listen tcp -screen 0 1024x768x24 > /dev/null & + export DISPLAY=:0 export PATH=/usr/bin:/usr/local/bin:$PATH + echo "Starting Ubuntu install script" - bash -xe ./installer/ubuntu/installer.sh - echo "Install finished" + installer/ubuntu/installer.sh + NLOPT_LIB=$(ls -d $HOME/.local/lib/python*/site-packages) echo $NLOPT_LIB export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$NLOPT_LIB export PYTHON=python3 + + # test X server xset -display $DISPLAY -q > /dev/null; fi install: - - if [[ ! "${TRAVIS_OS_NAME}" == "windows" ]]; then - make; - fi; - - | + - | # for mac build HNN .mod files + if [[ "${TRAVIS_OS_NAME}" == "osx" ]]; then + make -j2 + fi + - | # start the docker image build on linux if [[ "${TRAVIS_OS_NAME}" == "linux" ]]; then + make -j2 if [[ "$TRAVIS_PULL_REQUEST" =~ "false" ]]; then SOURCE_REPO="https://github.com/${TRAVIS_REPO_SLUG}.git" SOURCE_BRANCH=$TRAVIS_BRANCH @@ -202,16 +207,28 @@ install: --build-arg SOURCE_REPO=${SOURCE_REPO} \ installer/docker > docker_build.txt 2>&1 || echo "Failed build" >> docker_build.txt) & fi - - pip install flake8 pytest pytest-cov coverage coveralls mne + - | + pip install flake8 pytest pytest-cov coverage coveralls mne & + PIP_PID=$! + if [[ "${TRAVIS_OS_NAME}" == "windows" ]]; then + wsl -- pip install flake8 pytest pytest-cov coverage coveralls mne + fi + echo "Waiting for pip installation to finish..." + NAME="installing python test prerequisites" + wait_for_pid "${PIP_PID}" "$NAME" script: - - echo "Running Python tests" && py.test tests/ - - echo "Testing GUI on host OS" && $PYTHON hnn.py - - | + - | # Check that the GUI starts on host OS + echo "Testing GUI on host OS..." + $PYTHON hnn.py + - | # Run py.test that includes running a simulation and verifying results + echo "Running Python tests on host OS..." + py.test tests/ + - | # wait for docker image to be fully loaded into docker daemon if [[ "${TRAVIS_OS_NAME}" = "linux" ]]; then find "$HOME/hnn_out" -type d -exec chmod 777 {} \; && \ find "$HOME/hnn_out" -type f -exec chmod 666 {} \; - echo "Waiting on docker build:" + echo "Waiting on docker build..." cat docker_build.txt while ! cat docker_build.txt|grep "Successfully built"; do if cat docker_build.txt|grep "Failed build"; then @@ -223,36 +240,65 @@ script: done tail -1 docker_build.txt else + echo "Waiting for docker image to be loaded..." while ! test -e $HOME/docker_image_loaded; do - echo "Sleeping for 10s while waiting for docker image download to complete" - sleep 10 + sleep 1 done fi - - | - if [[ "${TRAVIS_OS_NAME}" = "windows" ]]; then - echo "Testing GUI in docker without ssh as user 'test user'" + - | # test HNN using Docker without SSH + echo "Testing GUI in docker without ssh..." + if [[ "${CONTAINER_TYPE}" = "windows" ]]; then + echo "Running as user 'test user'" powershell -executionpolicy bypass -File "scripts\start-test-as-user.ps1" else - echo "Testing GUI in docker without ssh" USE_SSH=0 ./hnn_docker.sh start || script_fail fi - - | + - | # Testing hnn_docker.sh upgrade command + echo "Testing hnn_docker.sh upgrade" if [[ "${TRAVIS_OS_NAME}" = "linux" ]]; then - echo "Testing MPI in docker" - make clean - # only run on linux because it's not virtualized through qemu - timeout 120 docker exec hnn_container bash -c "sudo apt-get install --no-install-recommends -y \ - make gcc libc6-dev libtinfo-dev libncurses-dev libx11-dev libreadline-dev" - timeout 300 docker exec -u $UID:hnn_group hnn_container bash -c "make && source /home/hnn_user/hnn_envs && \ - mpiexec -np 2 --oversubscribe nrniv -mpi -python run.py param/default.param && echo finished simulation" || { \ - echo -e "\nMPI test failed\n" - docker ps -a | grep hnn_container - false - } - echo "Testing hnn_docker.sh upgrade" && ./hnn_docker.sh upgrade || script_fail + ./hnn_docker.sh upgrade || script_fail + else + echo "Skipping" fi - - | - if [[ ! "${TRAVIS_OS_NAME}" = "windows" ]]; then - echo "Testing GUI in docker with ssh" && USE_SSH=1 ./hnn_docker.sh start || script_fail - echo "Testing hnn_docker.sh uninstall command" && ./hnn_docker.sh uninstall || script_fail + - | # For linux containers (mac/windows/linux OS), start HNN with docker using ssh + if [[ ! "${CONTAINER_TYPE}" = "windows" ]]; then + echo "Testing GUI in docker with ssh" + USE_SSH=1 ./hnn_docker.sh start || script_fail + fi + - | # test running a MPI simulation within container + if [[ $TRAVIS_OSX_IMAGE == "xcode8" ]] || [[ $TRAVIS_OSX_IMAGE == "xcode9.2" ]]; then + # tests on mac take a long time. sierra and el capitan are close to maximum build times + echo "Skipping" + else + if [[ "${TRAVIS_OS_NAME}" == "windows" ]]; then + find_command_suggested_path "vcxsrv" "/c/Program Files/VcXsrv" && \ + start_vcxsrv_print || script_fail + echo "Testing GUI on WSL..." + wsl -- DISPLAY=:0 python3 hnn.py || exit $? + echo "Building HNN in WSL..." + wsl -- bash -ec "cd /mnt/c/users/travis/build/jonescompneurolab/hnn && make clean && make -j2" || exit $? + echo "Running Python tests in WSL..." + wsl -- bash -ec "cd /mnt/c/users/travis/build/jonescompneurolab/hnn && DISPLAY=:0 py.test tests/" || exit $? + echo "Testing MPI in WSL..." + wsl -- bash -ec "cd /mnt/c/users/travis/build/jonescompneurolab/hnn/ && \ + source installer/docker/hnn_envs && \ + mpiexec --allow-run-as-root -np 2 nrniv -mpi -python run.py" || exit $? + stop_vcxsrv + else + echo "Testing MPI in docker..." + if [[ "$CONTAINER_TYPE" == "linux" ]]; then + timeout 120 docker exec hnn_container bash -c "sudo apt-get install --no-install-recommends -y \ + make gcc libc6-dev libtinfo-dev libncurses-dev libx11-dev libreadline-dev" + timeout 120 docker exec -u hnn_user hnn_container bash -c "make clean && make" + fi + timeout 300 docker exec -u hnn_user hnn_container bash -c "source /home/hnn_user/hnn_envs && \ + mpiexec -np 2 nrniv -mpi -python run.py && echo finished simulation" || { \ + echo -e "\nMPI test failed\n" + docker ps -a | grep hnn_container + false + } || script_fail + fi fi + - | # Test hnn_docker.sh uninstall command + echo "Testing hnn_docker.sh uninstall command" + ./hnn_docker.sh uninstall || script_fail diff --git a/scripts/docker_functions.sh b/scripts/docker_functions.sh index bc5964767..55f68c812 100644 --- a/scripts/docker_functions.sh +++ b/scripts/docker_functions.sh @@ -2353,6 +2353,11 @@ function get_docker_container_type { echo "$CONTAINER_TYPE" | tee -a "$LOGFILE" fi + if [[ "$CONTAINER_TYPE" == "windows" ]] && [[ "$OS" == "wsl" ]]; then + echo "** Windows containers not supported in WSL. Use Linux containers. **" | tee -a "$LOGFILE" + cleanup 1 + fi + export CONTAINER_TYPE } diff --git a/scripts/run-travis-windows.sh b/scripts/run-travis-windows.sh index b3a42ea47..8bdcf0100 100755 --- a/scripts/run-travis-windows.sh +++ b/scripts/run-travis-windows.sh @@ -29,4 +29,4 @@ export -f cleanup cd "$DIR/../" export PATH=$PATH:/C/tools/msys64/mingw64/bin -USE_SSH=0 "./hnn_docker.sh" start || script_fail \ No newline at end of file +USE_SSH=0 ./hnn_docker.sh start || script_fail diff --git a/scripts/setup-travis-windows.sh b/scripts/setup-travis-windows.sh index 80c1da0d7..e4546a68b 100755 --- a/scripts/setup-travis-windows.sh +++ b/scripts/setup-travis-windows.sh @@ -8,7 +8,8 @@ source "$DIR/utils.sh" # we use find_program_print and retry_docker_pull from docker_functions.sh source "$DIR/docker_functions.sh" -export LOGFILE="docker_setup.log" + +[[ $LOGFILE ]] || LOGFILE="hnn_travis.log" # docker_functions.sh expects some globals to be set set_globals @@ -32,12 +33,17 @@ function cleanup { } export -f cleanup -# start the docker pull in the background -find_program_print docker -(retry_docker_pull && touch $HOME/docker_image_loaded) & +echo "Installing Ubuntu WSL..." +powershell.exe -ExecutionPolicy Bypass -File ./scripts/setup-wsl.ps1 & +WSL_PID=$! + +# prepare for installing msys2 +[[ ! -f C:/tools/msys64/msys2_shell.cmd ]] && rm -rf C:/tools/msys64 +choco uninstall -y mingw # enable windows remoting service to log in as a different user to run tests powershell -Command 'Start-Service -Name WinRM' > /dev/null +powershell -Command 'Start-Service -Name seclogon' > /dev/null # change settings to allow a blank password for TEST_USER reg add HKEY_LOCAL_MACHINE\\SYSTEM\\CurrentControlSet\\Control\\Lsa //t REG_DWORD //v LimitBlankPasswordUse //d 0 //f 2>&1 > /dev/null @@ -58,51 +64,53 @@ runas //user:"$TEST_USER" "cmd /C whoami" > /dev/null < "$HOME/test_user_creds" # copy hnn source to test user's home directory TEST_USER_DIR="/c/Users/$TEST_USER" if [ -d "$TEST_USER_DIR" ]; then - mkdir "$TEST_USER_DIR/hnn" cp -r "$(pwd)" "$TEST_USER_DIR/" else echo "No user home directory created at $TEST_USER_DIR" exit 2 fi -echo "Starting Windows install script..." -powershell.exe -ExecutionPolicy Bypass -File ./installer/windows/hnn.ps1 & -POWERSHELL_PID=$! +echo "Installing Microsoft MPI" +powershell -command "(New-Object System.Net.WebClient).DownloadFile('https://github.com/microsoft/Microsoft-MPI/releases/download/v10.1.1/msmpisetup.exe', 'msmpisetup.exe')" && \ + ./msmpisetup.exe -unattend && \ + rm -f msmpisetup.exe -echo "Waiting for Windows install script to finish..." -NAME="installing HNN on host system" -wait_for_pid "${POWERSHELL_PID}" "$NAME" +echo "Running HNN Windows install script..." +powershell.exe -ExecutionPolicy Bypass -File ./installer/windows/hnn.ps1 +# add miniconda python to the path +export PATH=$PATH:$HOME/Miniconda3/Scripts +export PATH=$HOME/Miniconda3/envs/hnn/:$PATH +export PATH=$HOME/Miniconda3/envs/hnn/Scripts:$PATH +export PATH=$HOME/Miniconda3/envs/hnn/Library/bin:$PATH -URL="https://downloads.sourceforge.net/project/vcxsrv/vcxsrv/1.20.8.1/vcxsrv-64.1.20.8.1.installer.exe" -FILENAME="$HOME/vcxsrv-64.1.20.8.1.installer.exe" -start_download "$FILENAME" "$URL" > /dev/null & -VCXSRV_PID=$! - -# install msys2 to get opengl32.dll from mesa -# this is needed to be able to start vcxsrv for docker tests -[[ ! -f C:/tools/msys64/msys2_shell.cmd ]] && rm -rf C:/tools/msys64 -choco uninstall -y mingw +echo "Installing msys2 with choco..." +choco upgrade --no-progress -y msys2 &> /dev/null -# install vcxsrv for docker tests -echo "Waiting for VcXsrv download to finish..." -NAME="downloading VcXsrv" -wait_for_pid "${VCXSRV_PID}" "$NAME" +# start the docker pull in the background +echo "Starting to pull the docker image..." +find_program_print docker +((retry_docker_pull > /dev/null || script_fail) && touch $HOME/docker_image_loaded ) & -echo "Installing msys2 with choco..." -choco upgrade --no-progress -y msys2 & -MSYS2_PID=$! +echo "Downloading VcXsrv..." +URL="https://downloads.sourceforge.net/project/vcxsrv/vcxsrv/1.20.8.1/vcxsrv-64.1.20.8.1.installer.exe" +FILENAME="$HOME/vcxsrv-64.1.20.8.1.installer.exe" +start_download "$FILENAME" "$URL" > /dev/null echo "Installing VcXsrv..." cmd //c "$HOME/vcxsrv-64.1.20.8.1.installer.exe /S" -echo "Waiting for msys2 installation to finish..." -NAME="isntalling msys2" -wait_for_pid "${MSYS2_PID}" "$NAME" - +# get opengl32.dll from mesa +# this is needed to be able to start vcxsrv for docker tests export msys2='cmd //C RefreshEnv.cmd ' export msys2+='& set MSYS=winsymlinks:nativestrict ' export msys2+='& C:\\tools\\msys64\\msys2_shell.cmd -defterm -no-start' export mingw64="$msys2 -mingw64 -full-path -here -c "\"\$@"\" --" export msys2+=" -msys2 -c "\"\$@"\" --" -$msys2 pacman --sync --noconfirm --needed mingw-w64-x86_64-mesa & -# the command above will complete before the docker test begins +$msys2 pacman --sync --noconfirm --needed mingw-w64-x86_64-mesa + +echo "Downloading python test packages..." +pip download flake8 pytest pytest-cov coverage coveralls mne + +echo "Waiting for WSL install to finish..." +NAME="installing WSL" +wait_for_pid "${WSL_PID}" "$NAME" || script_fail diff --git a/scripts/setup-wsl.ps1 b/scripts/setup-wsl.ps1 new file mode 100644 index 000000000..92b0c12a9 --- /dev/null +++ b/scripts/setup-wsl.ps1 @@ -0,0 +1,26 @@ +$ErrorActionPreference = "Stop" + +Set-Location C:\Users\travis +Enable-WindowsOptionalFeature -Online -FeatureName Microsoft-Windows-Subsystem-Linux +Write-Host "Downloading Ubuntu image..." +Invoke-WebRequest -Uri https://aka.ms/wsl-ubuntu-1804 -OutFile Ubuntu.appx -UseBasicParsing + +Write-Host "Finished downloading Ubuntu image. Extracting..." +Move-Item .\Ubuntu.appx .\Ubuntu.zip +Expand-Archive .\Ubuntu.zip .\Ubuntu +$userenv = [System.Environment]::GetEnvironmentVariable("Path", "User"); [System.Environment]::SetEnvironmentVariable("PATH", $userenv + ";C:\Users\travis\Ubuntu", "User") + +Write-Host "Configuring Ubuntu WSL for $env:UserName..." +& .\Ubuntu\Ubuntu1804.exe install --root + +& wsl -- dos2unix.exe -n build/jonescompneurolab/hnn/installer/ubuntu/installer.sh ./installer-unix.sh +Write-Host "Installing HNN in Ubuntu WSL..." +Set-Location C:\Users\travis\build\jonescompneurolab\hnn +& wsl -- bash -c "export LOGFILE=/mnt/c/users/travis/build/jonescompneurolab/hnn/hnn_travis.log && TRAVIS_TESTING=1 /mnt/c/users/travis/installer-unix.sh" + +if (!$?) { + exit 1 +} +else { + exit 0 +} diff --git a/scripts/start-test-as-user.ps1 b/scripts/start-test-as-user.ps1 index 76aa731d9..cd03f4b54 100644 --- a/scripts/start-test-as-user.ps1 +++ b/scripts/start-test-as-user.ps1 @@ -1,9 +1,9 @@ $ErrorActionPreference = "Stop" ( Enable-PSRemoting -Force -SkipNetworkProfileCheck ) | out-null -winrm set winrm/config/service '@{AllowUnencrypted="true"}' -winrm set winrm/config/service/auth '@{Basic="true"}' -winrm set winrm/config/client '@{AllowUnencrypted="true"}' +(winrm set winrm/config/service '@{AllowUnencrypted="true"}' ) | out-null +(winrm set winrm/config/service/auth '@{Basic="true"}' ) | out-null +(winrm set winrm/config/client '@{AllowUnencrypted="true"}' ) | out-null # create credentials with no password ( $Credentials = [System.Management.Automation.PSCredential]::new("test user",[System.Security.SecureString]::new()) ) | out-null @@ -15,19 +15,31 @@ winrm set winrm/config/client '@{AllowUnencrypted="true"}' $session = New-PSSession -ComputerName localhost -SessionOption $opt -Authentication Basic -Credential $Credentials # Run the bash command as 'test user' -$env:ret = $false -Write-Output "Starting run-travis-windows.sh as user 'test user'" -Invoke-Command -Session $session -ScriptBlock { - $script:run_script="/c/users/$env:UserName/hnn/scripts/run-travis-windows.sh" - & "C:\Program Files\Git\bin\bash.exe" "$script:run_script" - Write-Output "Return status: $?" - $env:ret=$? +Write-Output "Starting run-travis-windows.sh as user 'test user'..." +try { + Invoke-Command -Session $session -ScriptBlock { + $script:run_script="/c/users/$env:UserName/hnn/scripts/run-travis-windows.sh" + try { + & "C:\Program Files\Git\bin\bash.exe" "$script:run_script" + if (!$?) { + Write-Output "run-travis-windows.sh returned $LastExitCode" + $host.SetShouldExit($LastExitCode) + exit $LastExitCode + } + } + catch { + Write-Output "Failed to run run-travis-windows.sh" + Write-Output $_ + $host.SetShouldExit(-1) + throw + } + } } - -if (!$env:ret) { +catch { + Write-Output "Failed to start session:" + Write-Output $_ $host.SetShouldExit(-1) - throw + exit -1 } -else { - exit 0 -} \ No newline at end of file + +Remove-PSSession -Session $session \ No newline at end of file diff --git a/scripts/utils.sh b/scripts/utils.sh index a82113c20..1bcf6c7f3 100755 --- a/scripts/utils.sh +++ b/scripts/utils.sh @@ -50,11 +50,10 @@ function wait_for_pid { } function script_fail { - # This function is specific to .travis.yml. always use hnn_docker.log and not - # $LOGFILE + check_var LOGFILE echo -ne "\n******* hnn_docker.sh failed. output from hnn_docker.log below *******\n" - cat hnn_docker.log + cat "$LOGFILE" exit 2 } From 09c49b4ec66526748f17ac1469f304fdd1fb0ecb Mon Sep 17 00:00:00 2001 From: Blake Caldwell Date: Fri, 15 May 2020 11:15:44 -0400 Subject: [PATCH 6/8] TST: start longest tests first on Travis --- .travis.yml | 99 +++++++++++++++++++++++++++-------------------------- 1 file changed, 50 insertions(+), 49 deletions(-) diff --git a/.travis.yml b/.travis.yml index 44ee632bb..cd34c959a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,10 +2,56 @@ language: c matrix: include: + # OSX + - os: osx + name: "MacOS sierra" + osx_image: xcode9.2 + env: + - PYTHON_VERSION=3.7 + - NEURON_VERSION=7.7 + - HOMEBREW_NO_AUTO_UPDATE=1 + - HOMEBREW_QEMU=1 + - HOMEBREW_SDKROOT= + + - os: osx + name: "MacOS el capitan" + osx_image: xcode8 + env: + - PYTHON_VERSION=3.7 + - NEURON_VERSION=7.7 + - HOMEBREW_NO_AUTO_UPDATE=1 + - BREW_UNTAP=1 + - HOMEBREW_QEMU=1 + - HOMEBREW_SDKROOT= + + - os: osx + name: "MacOS mojave" + osx_image: xcode11.3 + env: + - PYTHON_VERSION=3.7 + - NEURON_VERSION=7.7 + - HOMEBREW_NO_AUTO_UPDATE=1 + + - os: osx + name: "MacOS high sierra" + osx_image: xcode10.1 + env: + - PYTHON_VERSION=3.7 + - NEURON_VERSION=7.7 + - HOMEBREW_NO_AUTO_UPDATE=1 + - BUILD_QEMU=1 + + # Windows + - os: windows + name: "Windows" + env: + - PYTHON_VERSION=3.7 + - NEURON_VERSION=7.7 + # Linux - os: linux - dist: disco - name: "Ubuntu disco" + dist: xenial + name: "Ubuntu xenial" env: - NEURON_VERSION=7.7 services: @@ -30,8 +76,8 @@ matrix: - xvfb - os: linux - dist: xenial - name: "Ubuntu xenial" + dist: disco + name: "Ubuntu disco" env: - NEURON_VERSION=7.7 services: @@ -42,51 +88,6 @@ matrix: packages: - xvfb - # OSX - - os: osx - name: "MacOS mojave" - osx_image: xcode11.3 - env: - - PYTHON_VERSION=3.7 - - NEURON_VERSION=7.7 - - HOMEBREW_NO_AUTO_UPDATE=1 - - - os: osx - name: "MacOS high sierra" - osx_image: xcode10.1 - env: - - PYTHON_VERSION=3.7 - - NEURON_VERSION=7.7 - - HOMEBREW_NO_AUTO_UPDATE=1 - - BUILD_QEMU=1 - - - os: osx - name: "MacOS sierra" - osx_image: xcode9.2 - env: - - PYTHON_VERSION=3.7 - - NEURON_VERSION=7.7 - - HOMEBREW_NO_AUTO_UPDATE=1 - - HOMEBREW_QEMU=1 - - HOMEBREW_SDKROOT= - - - os: osx - name: "MacOS el capitan" - osx_image: xcode8 - env: - - PYTHON_VERSION=3.7 - - NEURON_VERSION=7.7 - - HOMEBREW_NO_AUTO_UPDATE=1 - - BREW_UNTAP=1 - - HOMEBREW_QEMU=1 - - HOMEBREW_SDKROOT= - - # Windows - - os: windows - name: "Windows" - env: - - PYTHON_VERSION=3.7 - - NEURON_VERSION=7.7 before_install: - set -e # error on any command failure From 73dcd71e4cb34d30c010ffea8b3b7492b57040a2 Mon Sep 17 00:00:00 2001 From: Blake Caldwell Date: Fri, 15 May 2020 11:10:45 -0400 Subject: [PATCH 7/8] TST: run wsl commands as hnn_user --- .travis.yml | 13 ++---------- scripts/run-travis-wsl.sh | 21 +++++++++++++++++++ scripts/setup-travis-windows.sh | 5 ++++- scripts/setup-travis-wsl.ps1 | 36 +++++++++++++++++++++++++++++++++ scripts/setup-wsl.ps1 | 26 ------------------------ 5 files changed, 63 insertions(+), 38 deletions(-) create mode 100755 scripts/run-travis-wsl.sh create mode 100644 scripts/setup-travis-wsl.ps1 delete mode 100644 scripts/setup-wsl.ps1 diff --git a/.travis.yml b/.travis.yml index cd34c959a..0275b0b70 100644 --- a/.travis.yml +++ b/.travis.yml @@ -274,17 +274,8 @@ script: if [[ "${TRAVIS_OS_NAME}" == "windows" ]]; then find_command_suggested_path "vcxsrv" "/c/Program Files/VcXsrv" && \ start_vcxsrv_print || script_fail - echo "Testing GUI on WSL..." - wsl -- DISPLAY=:0 python3 hnn.py || exit $? - echo "Building HNN in WSL..." - wsl -- bash -ec "cd /mnt/c/users/travis/build/jonescompneurolab/hnn && make clean && make -j2" || exit $? - echo "Running Python tests in WSL..." - wsl -- bash -ec "cd /mnt/c/users/travis/build/jonescompneurolab/hnn && DISPLAY=:0 py.test tests/" || exit $? - echo "Testing MPI in WSL..." - wsl -- bash -ec "cd /mnt/c/users/travis/build/jonescompneurolab/hnn/ && \ - source installer/docker/hnn_envs && \ - mpiexec --allow-run-as-root -np 2 nrniv -mpi -python run.py" || exit $? - stop_vcxsrv + wsl -- //home/hnn_user/hnn/scripts/run-travis-wsl.sh + stop_vcxsrv || script_fail else echo "Testing MPI in docker..." if [[ "$CONTAINER_TYPE" == "linux" ]]; then diff --git a/scripts/run-travis-wsl.sh b/scripts/run-travis-wsl.sh new file mode 100755 index 000000000..642d76ed1 --- /dev/null +++ b/scripts/run-travis-wsl.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +export TRAVIS_TESTING=1 + +source "$DIR/../installer/docker/hnn_envs" + +echo "Testing GUI on WSL..." +cd $DIR/../ +export DISPLAY=:0 +python3 hnn.py + +echo "Building HNN in WSL..." +make clean +make -j2 + +echo "Running Python tests in WSL..." +py.test tests/ + +echo "Testing MPI in WSL..." +mpiexec -np 2 nrniv -mpi -python run.py \ No newline at end of file diff --git a/scripts/setup-travis-windows.sh b/scripts/setup-travis-windows.sh index e4546a68b..3ed2872bb 100755 --- a/scripts/setup-travis-windows.sh +++ b/scripts/setup-travis-windows.sh @@ -34,7 +34,7 @@ function cleanup { export -f cleanup echo "Installing Ubuntu WSL..." -powershell.exe -ExecutionPolicy Bypass -File ./scripts/setup-wsl.ps1 & +powershell.exe -ExecutionPolicy Bypass -File ./scripts/setup-travis-wsl.ps1 & WSL_PID=$! # prepare for installing msys2 @@ -111,6 +111,9 @@ $msys2 pacman --sync --noconfirm --needed mingw-w64-x86_64-mesa echo "Downloading python test packages..." pip download flake8 pytest pytest-cov coverage coveralls mne +echo "Downloading python test packages in WSL (command may fail)..." +wsl -- pip download flake8 pytest pytest-cov coverage coveralls mne || true + echo "Waiting for WSL install to finish..." NAME="installing WSL" wait_for_pid "${WSL_PID}" "$NAME" || script_fail diff --git a/scripts/setup-travis-wsl.ps1 b/scripts/setup-travis-wsl.ps1 new file mode 100644 index 000000000..86ec9ba7f --- /dev/null +++ b/scripts/setup-travis-wsl.ps1 @@ -0,0 +1,36 @@ +$ErrorActionPreference = "Stop" + +Set-Location C:\Users\travis +Enable-WindowsOptionalFeature -Online -FeatureName Microsoft-Windows-Subsystem-Linux +Write-Host "Downloading Ubuntu image..." +Invoke-WebRequest -Uri https://aka.ms/wsl-ubuntu-1804 -OutFile Ubuntu.appx -UseBasicParsing + +Write-Host "Finished downloading Ubuntu image. Extracting..." +Move-Item .\Ubuntu.appx .\Ubuntu.zip +Expand-Archive .\Ubuntu.zip .\Ubuntu +$userenv = [System.Environment]::GetEnvironmentVariable("Path", "User"); [System.Environment]::SetEnvironmentVariable("PATH", $userenv + ";C:\Users\travis\Ubuntu", "User") + +Write-Host "Configuring Ubuntu WSL..." +& .\Ubuntu\Ubuntu1804.exe install --root + +# add hnn_user +& wsl -- bash -ec "groupadd hnn_group && useradd -m -b /home/ -g hnn_group hnn_user && adduser hnn_user sudo && echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers && chsh -s /bin/bash hnn_user" + +# copy hnn dir to hnn_user homedir and change permissions +& wsl -- bash -ec "cp -r build/jonescompneurolab/hnn /home/hnn_user/ && chown -R hnn_user: /home/hnn_user && apt-get update && apt-get install -y dos2unix" + +# run future commands as hnn_user +& .\Ubuntu\Ubuntu1804.exe config --default-user hnn_user + +# remove windows newlines +& wsl -- bash -ec "dos2unix /home/hnn_user/hnn/scripts/* /home/hnn_user/hnn/installer/ubuntu/installer.sh /home/hnn_user/hnn/installer/docker/hnn_envs" + +Write-Host "Installing HNN in Ubuntu WSL..." +& wsl -- bash -ec "cd /home/hnn_user/hnn && source scripts/utils.sh && export LOGFILE=ubuntu_install.log && TRAVIS_TESTING=1 installer/ubuntu/installer.sh || script_fail" + +if (!$?) { + exit 1 +} +else { + exit 0 +} diff --git a/scripts/setup-wsl.ps1 b/scripts/setup-wsl.ps1 deleted file mode 100644 index 92b0c12a9..000000000 --- a/scripts/setup-wsl.ps1 +++ /dev/null @@ -1,26 +0,0 @@ -$ErrorActionPreference = "Stop" - -Set-Location C:\Users\travis -Enable-WindowsOptionalFeature -Online -FeatureName Microsoft-Windows-Subsystem-Linux -Write-Host "Downloading Ubuntu image..." -Invoke-WebRequest -Uri https://aka.ms/wsl-ubuntu-1804 -OutFile Ubuntu.appx -UseBasicParsing - -Write-Host "Finished downloading Ubuntu image. Extracting..." -Move-Item .\Ubuntu.appx .\Ubuntu.zip -Expand-Archive .\Ubuntu.zip .\Ubuntu -$userenv = [System.Environment]::GetEnvironmentVariable("Path", "User"); [System.Environment]::SetEnvironmentVariable("PATH", $userenv + ";C:\Users\travis\Ubuntu", "User") - -Write-Host "Configuring Ubuntu WSL for $env:UserName..." -& .\Ubuntu\Ubuntu1804.exe install --root - -& wsl -- dos2unix.exe -n build/jonescompneurolab/hnn/installer/ubuntu/installer.sh ./installer-unix.sh -Write-Host "Installing HNN in Ubuntu WSL..." -Set-Location C:\Users\travis\build\jonescompneurolab\hnn -& wsl -- bash -c "export LOGFILE=/mnt/c/users/travis/build/jonescompneurolab/hnn/hnn_travis.log && TRAVIS_TESTING=1 /mnt/c/users/travis/installer-unix.sh" - -if (!$?) { - exit 1 -} -else { - exit 0 -} From 57a305b6f9e7a896e90ab94c4bd44f3e82766af1 Mon Sep 17 00:00:00 2001 From: Blake Caldwell Date: Fri, 15 May 2020 13:59:15 -0400 Subject: [PATCH 8/8] MAINT: rename install scripts Use a hyphen instead of underscore in install script names --- .travis.yml | 2 +- installer/centos/{centos6-installer.sh => hnn-centos6.sh} | 0 installer/centos/{centos7-installer.sh => hnn-centos7.sh} | 0 installer/centos/native_install.md | 8 ++++---- installer/ubuntu/{installer.sh => hnn-ubuntu.sh} | 0 installer/windows/Dockerfile | 4 ++-- installer/windows/{hnn.ps1 => hnn-windows.ps1} | 0 installer/windows/native_install.md | 8 ++++---- scripts/setup-travis-windows.sh | 2 +- scripts/setup-travis-wsl.ps1 | 4 ++-- 10 files changed, 14 insertions(+), 14 deletions(-) rename installer/centos/{centos6-installer.sh => hnn-centos6.sh} (100%) rename installer/centos/{centos7-installer.sh => hnn-centos7.sh} (100%) rename installer/ubuntu/{installer.sh => hnn-ubuntu.sh} (100%) rename installer/windows/{hnn.ps1 => hnn-windows.ps1} (100%) diff --git a/.travis.yml b/.travis.yml index 0275b0b70..3ddd6134d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -175,7 +175,7 @@ before_install: export PATH=/usr/bin:/usr/local/bin:$PATH echo "Starting Ubuntu install script" - installer/ubuntu/installer.sh + installer/ubuntu/hnn-ubuntu.sh NLOPT_LIB=$(ls -d $HOME/.local/lib/python*/site-packages) echo $NLOPT_LIB diff --git a/installer/centos/centos6-installer.sh b/installer/centos/hnn-centos6.sh similarity index 100% rename from installer/centos/centos6-installer.sh rename to installer/centos/hnn-centos6.sh diff --git a/installer/centos/centos7-installer.sh b/installer/centos/hnn-centos7.sh similarity index 100% rename from installer/centos/centos7-installer.sh rename to installer/centos/hnn-centos7.sh diff --git a/installer/centos/native_install.md b/installer/centos/native_install.md index 991a895e0..00cd5dd2a 100644 --- a/installer/centos/native_install.md +++ b/installer/centos/native_install.md @@ -5,15 +5,15 @@ The script below assumes that it can update OS packages for python and prerequis - CentOS 7: [centos7-installer.sh](centos7-installer.sh) ```bash - chmod +x ./centos7-installer.sh - ./centos7-installer.sh + chmod +x ./hnn-centos7.sh + ./hnn-centos7.sh ``` - CentOS 6 (no longer maintained): [centos6-installer.sh](centos6-installer.sh) ```bash - chmod +x ./centos6-installer.sh - ./centos6-installer.sh + chmod +x ./hnn-centos6.sh + ./hnn-centos6.sh ``` ## Troubleshooting diff --git a/installer/ubuntu/installer.sh b/installer/ubuntu/hnn-ubuntu.sh similarity index 100% rename from installer/ubuntu/installer.sh rename to installer/ubuntu/hnn-ubuntu.sh diff --git a/installer/windows/Dockerfile b/installer/windows/Dockerfile index a77391d43..8cfd878fb 100644 --- a/installer/windows/Dockerfile +++ b/installer/windows/Dockerfile @@ -28,8 +28,8 @@ COPY installer/docker/hnn_envs /home/hnn_user COPY installer/docker/start_hnn.sh /home/hnn_user COPY installer/docker/check_hnn_out_perms.sh /home/hnn_user -ADD installer/windows/hnn.ps1 /home/hnn_user/Downloads/ -RUN powershell -executionpolicy bypass -File "C:\home\hnn_user\Downloads\hnn.ps1" && \ +ADD installer/windows/hnn-windows.ps1 /home/hnn_user/Downloads/ +RUN powershell -executionpolicy bypass -File "C:\home\hnn_user\Downloads\hnn-windows.ps1" && \ powershell -Command "$env:PATH = 'C:\home\hnn_user\AppData\Local\Programs\Git\bin;' + $env:PATH + \ ';"C:\Program Files\VcXsrv";C:\home\hnn_user\Miniconda3\Scripts;C:\home\hnn_user\Miniconda3\envs\hnn;' + \ 'C:\home\hnn_user\Miniconda3\envs\hnn\Scripts;C:\home\hnn_user\Miniconda3\envs\hnn\Library\bin' ; \ diff --git a/installer/windows/hnn.ps1 b/installer/windows/hnn-windows.ps1 similarity index 100% rename from installer/windows/hnn.ps1 rename to installer/windows/hnn-windows.ps1 diff --git a/installer/windows/native_install.md b/installer/windows/native_install.md index 4ad25aa0b..20ecd9383 100644 --- a/installer/windows/native_install.md +++ b/installer/windows/native_install.md @@ -2,7 +2,7 @@ This method will run HNN without using virtualization, meaning the GUI may feel more responsive and simulations may run slightly faster. However, there is a possibility that differences in the base environment may require additional troubleshooting. Thus, it is best suited for advanced users, and we recommend the [Docker-based install instructions](README.md). -With this method, a [PowerShell install script](hnn.ps1) will manage downloading all prerequisites, including Python (Anaconda), NEURON, and Git for Windows. +With this method, a [PowerShell install script](hnn-windows.ps1) will manage downloading all prerequisites, including Python (Anaconda), NEURON, and Git for Windows. ## Requirements @@ -20,19 +20,19 @@ The PowerShell script used below will create a new directory called "hnn" in the ```powershell cd hnn - powershell.exe -ExecutionPolicy Bypass -File .\installer\windows\hnn.ps1 + powershell.exe -ExecutionPolicy Bypass -File .\installer\windows\hnn-windows.ps1 ``` OR to download and run the script from a url: ```powershell - @"%SystemRoot%\System32\WindowsPowerShell\v1.0\powershell.exe" -NoProfile -InputFormat None -ExecutionPolicy Bypass -Command "iex ((New-Object System.Net.WebClient).DownloadString('https://raw.githubusercontent.com/jonescompneurolab/hnn/master/installer/windows/hnn.ps1'))" + @"%SystemRoot%\System32\WindowsPowerShell\v1.0\powershell.exe" -NoProfile -InputFormat None -ExecutionPolicy Bypass -Command "iex ((New-Object System.Net.WebClient).DownloadString('https://raw.githubusercontent.com/jonescompneurolab/hnn/master/installer/windows/hnn-windows.ps1'))" ``` OR from a powershell prompt: ```powershell - Set-ExecutionPolicy Bypass -Scope Process -Force; iex ((New-Object System.Net.WebClient).DownloadString('https://raw.githubusercontent.com/jonescompneurolab/hnn/master/installer/windows/hnn.ps1')) + Set-ExecutionPolicy Bypass -Scope Process -Force; iex ((New-Object System.Net.WebClient).DownloadString('https://raw.githubusercontent.com/jonescompneurolab/hnn/master/installer/windows/hnn-windows.ps1')) ``` - There will be a permission prompt to install Microsoft MPI and a couple of terminal windows will diff --git a/scripts/setup-travis-windows.sh b/scripts/setup-travis-windows.sh index 3ed2872bb..625f3ea6c 100755 --- a/scripts/setup-travis-windows.sh +++ b/scripts/setup-travis-windows.sh @@ -76,7 +76,7 @@ powershell -command "(New-Object System.Net.WebClient).DownloadFile('https://git rm -f msmpisetup.exe echo "Running HNN Windows install script..." -powershell.exe -ExecutionPolicy Bypass -File ./installer/windows/hnn.ps1 +powershell.exe -ExecutionPolicy Bypass -File ./installer/windows/hnn-windows.ps1 # add miniconda python to the path export PATH=$PATH:$HOME/Miniconda3/Scripts export PATH=$HOME/Miniconda3/envs/hnn/:$PATH diff --git a/scripts/setup-travis-wsl.ps1 b/scripts/setup-travis-wsl.ps1 index 86ec9ba7f..cd0d4a2f7 100644 --- a/scripts/setup-travis-wsl.ps1 +++ b/scripts/setup-travis-wsl.ps1 @@ -23,10 +23,10 @@ Write-Host "Configuring Ubuntu WSL..." & .\Ubuntu\Ubuntu1804.exe config --default-user hnn_user # remove windows newlines -& wsl -- bash -ec "dos2unix /home/hnn_user/hnn/scripts/* /home/hnn_user/hnn/installer/ubuntu/installer.sh /home/hnn_user/hnn/installer/docker/hnn_envs" +& wsl -- bash -ec "dos2unix /home/hnn_user/hnn/scripts/* /home/hnn_user/hnn/installer/ubuntu/hnn-ubuntu.sh /home/hnn_user/hnn/installer/docker/hnn_envs" Write-Host "Installing HNN in Ubuntu WSL..." -& wsl -- bash -ec "cd /home/hnn_user/hnn && source scripts/utils.sh && export LOGFILE=ubuntu_install.log && TRAVIS_TESTING=1 installer/ubuntu/installer.sh || script_fail" +& wsl -- bash -ec "cd /home/hnn_user/hnn && source scripts/utils.sh && export LOGFILE=ubuntu_install.log && TRAVIS_TESTING=1 installer/ubuntu/hnn-ubuntu.sh || script_fail" if (!$?) { exit 1