diff --git a/.travis.yml b/.travis.yml index b623ab2f5..3ddd6134d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,10 +2,56 @@ language: c matrix: include: + # OSX + - os: osx + name: "MacOS sierra" + osx_image: xcode9.2 + env: + - PYTHON_VERSION=3.7 + - NEURON_VERSION=7.7 + - HOMEBREW_NO_AUTO_UPDATE=1 + - HOMEBREW_QEMU=1 + - HOMEBREW_SDKROOT= + + - os: osx + name: "MacOS el capitan" + osx_image: xcode8 + env: + - PYTHON_VERSION=3.7 + - NEURON_VERSION=7.7 + - HOMEBREW_NO_AUTO_UPDATE=1 + - BREW_UNTAP=1 + - HOMEBREW_QEMU=1 + - HOMEBREW_SDKROOT= + + - os: osx + name: "MacOS mojave" + osx_image: xcode11.3 + env: + - PYTHON_VERSION=3.7 + - NEURON_VERSION=7.7 + - HOMEBREW_NO_AUTO_UPDATE=1 + + - os: osx + name: "MacOS high sierra" + osx_image: xcode10.1 + env: + - PYTHON_VERSION=3.7 + - NEURON_VERSION=7.7 + - HOMEBREW_NO_AUTO_UPDATE=1 + - BUILD_QEMU=1 + + # Windows + - os: windows + name: "Windows" + env: + - PYTHON_VERSION=3.7 + - NEURON_VERSION=7.7 + # Linux - os: linux - dist: disco - name: "Ubuntu disco" + dist: xenial + name: "Ubuntu xenial" env: - NEURON_VERSION=7.7 services: @@ -30,8 +76,8 @@ matrix: - xvfb - os: linux - dist: xenial - name: "Ubuntu xenial" + dist: disco + name: "Ubuntu disco" env: - NEURON_VERSION=7.7 services: @@ -42,96 +88,36 @@ matrix: packages: - xvfb - # OSX - - os: osx - name: "MacOS mojave" - osx_image: xcode11.3 - env: - - PYTHON_VERSION=3.7 - - NEURON_VERSION=7.7 - - HOMEBREW_NO_AUTO_UPDATE=1 - - - os: osx - name: "MacOS high sierra" - osx_image: xcode10.1 - env: - - PYTHON_VERSION=3.7 - - NEURON_VERSION=7.7 - - HOMEBREW_NO_AUTO_UPDATE=1 - - BUILD_QEMU=1 - - - os: osx - name: "MacOS sierra" - osx_image: xcode9.2 - env: - - PYTHON_VERSION=3.7 - - NEURON_VERSION=7.7 - - HOMEBREW_NO_AUTO_UPDATE=1 - - HOMEBREW_QEMU=1 - - HOMEBREW_SDKROOT= - - - os: osx - name: "MacOS el capitan" - osx_image: xcode8 - env: - - PYTHON_VERSION=3.7 - - NEURON_VERSION=7.7 - - HOMEBREW_NO_AUTO_UPDATE=1 - - BREW_UNTAP=1 - - HOMEBREW_QEMU=1 - - HOMEBREW_SDKROOT= - - # Windows - - os: windows - name: "Windows" - env: - - PYTHON_VERSION=3.7 - - NEURON_VERSION=7.7 before_install: - - set -e - - | + - set -e # error on any command failure + - | # function exports export TRAVIS_TESTING=1 - source scripts/utils.sh - source scripts/docker_functions.sh + # source utility functions export LOGFILE="hnn_travis.log" - set_globals - - # override cleanup function in docker_functions.sh - function cleanup { - check_var LOGFILE - - local __failed - - __failed=$1 - - echo -e "\n=====================" >> $LOGFILE - echo "cleanup() called from: ${FUNCNAME[1]} (L:${BASH_LINENO[0]})" >> $LOGFILE - - if [[ $__failed -ne "0" ]]; then - echo -e "\n======================================" - echo "Error: Please see log output for more details" - cat $LOGFILE - return $__failed - fi - } - + source scripts/docker_functions.sh + source scripts/utils.sh export -f cleanup - export DOCKER_IMAGE_NAME="jonescompneurolab/hnn:master" - export BASE_QEMU_OPTS="--disable-cocoa --disable-curses --disable-vnc --disable-vde \ - --disable-pie --disable-libusb --disable-hax --disable-kvm \ - --disable-debug-info --disable-docs --disable-nettle \ - --disable-sparse --disable-guest-agent --disable-qom-cast-debug \ - --disable-lzo --disable-bzip2 --disable-fdt \ - --disable-tpm --disable-replication --disable-modules --disable-blobs \ - --disable-gnutls --disable-vhost-crypto --disable-live-block-migration \ - --disable-user --disable-vhost-net" + # some functions require $docker_cmd and $CONTAINER_TYPE + set_globals - | if [ "${TRAVIS_OS_NAME}" == "osx" ]; then # install osx prerequisites + echo "Installing macOS prerequisites" + scripts/setup-travis-mac.sh + export PATH=${HOME}/miniconda/bin:$PATH + export PATH=$PATH:/Applications/NEURON-${NEURON_VERSION}/nrn/x86_64/bin + export PYTHONPATH=/Applications/NEURON-${NEURON_VERSION}/nrn/lib/python:$PYTHONPATH + export PATH=$PATH:/Applications/Docker.app/Contents/Resources/bin + export PYTHON=python3 + eval "$(docker-machine env default)" + + # check container type + find_program_print docker && export docker_cmd || script_fail + get_docker_container_type && export CONTAINER_TYPE || script_fail # set up DISPLAY and make sure that xquartz is ready check_xquartz_listening || { @@ -141,17 +127,16 @@ before_install: # accept TCP connections in hnn_docker.sh later xset -display $DISPLAY -q > /dev/null - export PATH=${HOME}/miniconda/bin:$PATH - export PATH=$PATH:/Applications/NEURON-${NEURON_VERSION}/nrn/x86_64/bin - export PYTHONPATH=/Applications/NEURON-${NEURON_VERSION}/nrn/lib/python:$PYTHONPATH - export PATH=$PATH:/Applications/Docker.app/Contents/Resources/bin - export PYTHON=python3 - eval "$(docker-machine env default)" - source activate hnn && echo "activated conda HNN environment" fi - - | - if [ "${TRAVIS_OS_NAME}" == "windows" ]; then # install windows prerequisites + - | # windows + if [ "${TRAVIS_OS_NAME}" == "windows" ]; then + echo "Installing windows prerequisites" + + # check container type + find_program_print docker && export docker_cmd || script_fail + get_docker_container_type && export CONTAINER_TYPE || script_fail + scripts/setup-travis-windows.sh # add miniconda python to the path @@ -160,34 +145,55 @@ before_install: export PATH=$HOME/Miniconda3/envs/hnn/Scripts:$PATH export PATH=$HOME/Miniconda3/envs/hnn/Library/bin:$PATH + # for using X server + export PATH="$PATH:/c/Program\ Files/VcXsrv" + + # for MESA dll's + export PATH=$PATH:/c/tools/msys64/mingw64/bin + + # for sharing with WSL environment + export WSLENV=TRAVIS_TESTING/u + # set other variables for neuron and HNN export PATH=$PATH:/c/nrn/bin export DISPLAY="localhost:0" export NEURONHOME=/c/nrn export PYTHON=python fi - - | + - | # Linux if [ "${TRAVIS_OS_NAME}" == "linux" ]; then + echo "Installing Linux prerequisites" + + # check container type + find_program_print docker && export docker_cmd || script_fail + get_docker_container_type && export CONTAINER_TYPE || script_fail + echo "Starting fake Xserver" Xvfb :0 -listen tcp -screen 0 1024x768x24 > /dev/null & + export DISPLAY=:0 export PATH=/usr/bin:/usr/local/bin:$PATH + echo "Starting Ubuntu install script" - bash -xe ./installer/ubuntu/installer.sh - echo "Install finished" + installer/ubuntu/hnn-ubuntu.sh + NLOPT_LIB=$(ls -d $HOME/.local/lib/python*/site-packages) echo $NLOPT_LIB export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$NLOPT_LIB export PYTHON=python3 + + # test X server xset -display $DISPLAY -q > /dev/null; fi install: - - if [[ ! "${TRAVIS_OS_NAME}" == "windows" ]]; then - make; - fi; - - | + - | # for mac build HNN .mod files + if [[ "${TRAVIS_OS_NAME}" == "osx" ]]; then + make -j2 + fi + - | # start the docker image build on linux if [[ "${TRAVIS_OS_NAME}" == "linux" ]]; then + make -j2 if [[ "$TRAVIS_PULL_REQUEST" =~ "false" ]]; then SOURCE_REPO="https://github.com/${TRAVIS_REPO_SLUG}.git" SOURCE_BRANCH=$TRAVIS_BRANCH @@ -202,16 +208,28 @@ install: --build-arg SOURCE_REPO=${SOURCE_REPO} \ installer/docker > docker_build.txt 2>&1 || echo "Failed build" >> docker_build.txt) & fi - - pip install flake8 pytest pytest-cov coverage coveralls mne + - | + pip install flake8 pytest pytest-cov coverage coveralls mne & + PIP_PID=$! + if [[ "${TRAVIS_OS_NAME}" == "windows" ]]; then + wsl -- pip install flake8 pytest pytest-cov coverage coveralls mne + fi + echo "Waiting for pip installation to finish..." + NAME="installing python test prerequisites" + wait_for_pid "${PIP_PID}" "$NAME" script: - - echo "Running Python tests" && py.test tests/ - - echo "Testing GUI on host OS" && $PYTHON hnn.py - - | + - | # Check that the GUI starts on host OS + echo "Testing GUI on host OS..." + $PYTHON hnn.py + - | # Run py.test that includes running a simulation and verifying results + echo "Running Python tests on host OS..." + py.test tests/ + - | # wait for docker image to be fully loaded into docker daemon if [[ "${TRAVIS_OS_NAME}" = "linux" ]]; then find "$HOME/hnn_out" -type d -exec chmod 777 {} \; && \ find "$HOME/hnn_out" -type f -exec chmod 666 {} \; - echo "Waiting on docker build:" + echo "Waiting on docker build..." cat docker_build.txt while ! cat docker_build.txt|grep "Successfully built"; do if cat docker_build.txt|grep "Failed build"; then @@ -223,36 +241,56 @@ script: done tail -1 docker_build.txt else + echo "Waiting for docker image to be loaded..." while ! test -e $HOME/docker_image_loaded; do - echo "Sleeping for 10s while waiting for docker image download to complete" - sleep 10 + sleep 1 done fi - - | - if [[ "${TRAVIS_OS_NAME}" = "windows" ]]; then - echo "Testing GUI in docker without ssh as user 'test user'" + - | # test HNN using Docker without SSH + echo "Testing GUI in docker without ssh..." + if [[ "${CONTAINER_TYPE}" = "windows" ]]; then + echo "Running as user 'test user'" powershell -executionpolicy bypass -File "scripts\start-test-as-user.ps1" else - echo "Testing GUI in docker without ssh" USE_SSH=0 ./hnn_docker.sh start || script_fail fi - - | + - | # Testing hnn_docker.sh upgrade command + echo "Testing hnn_docker.sh upgrade" if [[ "${TRAVIS_OS_NAME}" = "linux" ]]; then - echo "Testing MPI in docker" - make clean - # only run on linux because it's not virtualized through qemu - timeout 120 docker exec hnn_container bash -c "sudo apt-get install --no-install-recommends -y \ - make gcc libc6-dev libtinfo-dev libncurses-dev libx11-dev libreadline-dev" - timeout 300 docker exec -u $UID:hnn_group hnn_container bash -c "make && source /home/hnn_user/hnn_envs && \ - mpiexec -np 2 --oversubscribe nrniv -mpi -python run.py param/default.param && echo finished simulation" || { \ - echo -e "\nMPI test failed\n" - docker ps -a | grep hnn_container - false - } - echo "Testing hnn_docker.sh upgrade" && ./hnn_docker.sh upgrade || script_fail + ./hnn_docker.sh upgrade || script_fail + else + echo "Skipping" fi - - | - if [[ ! "${TRAVIS_OS_NAME}" = "windows" ]]; then - echo "Testing GUI in docker with ssh" && USE_SSH=1 ./hnn_docker.sh start || script_fail - echo "Testing hnn_docker.sh uninstall command" && ./hnn_docker.sh uninstall || script_fail + - | # For linux containers (mac/windows/linux OS), start HNN with docker using ssh + if [[ ! "${CONTAINER_TYPE}" = "windows" ]]; then + echo "Testing GUI in docker with ssh" + USE_SSH=1 ./hnn_docker.sh start || script_fail + fi + - | # test running a MPI simulation within container + if [[ $TRAVIS_OSX_IMAGE == "xcode8" ]] || [[ $TRAVIS_OSX_IMAGE == "xcode9.2" ]]; then + # tests on mac take a long time. sierra and el capitan are close to maximum build times + echo "Skipping" + else + if [[ "${TRAVIS_OS_NAME}" == "windows" ]]; then + find_command_suggested_path "vcxsrv" "/c/Program Files/VcXsrv" && \ + start_vcxsrv_print || script_fail + wsl -- //home/hnn_user/hnn/scripts/run-travis-wsl.sh + stop_vcxsrv || script_fail + else + echo "Testing MPI in docker..." + if [[ "$CONTAINER_TYPE" == "linux" ]]; then + timeout 120 docker exec hnn_container bash -c "sudo apt-get install --no-install-recommends -y \ + make gcc libc6-dev libtinfo-dev libncurses-dev libx11-dev libreadline-dev" + timeout 120 docker exec -u hnn_user hnn_container bash -c "make clean && make" + fi + timeout 300 docker exec -u hnn_user hnn_container bash -c "source /home/hnn_user/hnn_envs && \ + mpiexec -np 2 nrniv -mpi -python run.py && echo finished simulation" || { \ + echo -e "\nMPI test failed\n" + docker ps -a | grep hnn_container + false + } || script_fail + fi fi + - | # Test hnn_docker.sh uninstall command + echo "Testing hnn_docker.sh uninstall command" + ./hnn_docker.sh uninstall || script_fail diff --git a/hnn_docker.sh b/hnn_docker.sh index a5c3eb5c3..7e174551e 100755 --- a/hnn_docker.sh +++ b/hnn_docker.sh @@ -10,9 +10,14 @@ cleanup 1 } +[[ $LOGFILE ]] || export LOGFILE="$(pwd)/hnn_docker.log" source scripts/docker_functions.sh -export LOGFILE="$(pwd)/hnn_docker.log" + set_globals +if [[ $? -ne 0 ]]; then + echo "*failed*" | tee -a "$LOGFILE" + cleanup 2 +fi echo -e "======================================" while [ -n "$1" ]; do @@ -41,7 +46,7 @@ print_header_message "Checking OS version... " echo "$OS" | tee -a "$LOGFILE" # defaults -if [[ "$OS" =~ "linux" ]]; then +if [[ "$OS" == "linux" ]] || [[ "$OS" == "wsl" ]]; then [[ $USE_SSH ]] || USE_SSH=0 else [[ $USE_SSH ]] || USE_SSH=1 @@ -77,19 +82,25 @@ if [[ $? -ne "0" ]]; then fi fi -if [[ $UPGRADE -eq "1" ]]; then +get_docker_container_type +if [[ $? -ne "0" ]]; then + exit $? +fi +# CONTAINER_TYPE is set + +if [[ $UPGRADE -eq 1 ]]; then retry_docker_pull - if [[ $? -ne "0" ]]; then + if [[ $? -ne 0 ]]; then cleanup 2 fi - if [[ $START -eq "0" ]]; then + if [[ $START -eq 0 ]]; then # just doing upgrade cleanup 0 fi fi -if [[ "$STOP" -eq "1" ]]; then +if [[ $STOP -eq 1 ]]; then stop_container_fail if [[ "${DOCKER_TOOLBOX}" -eq "1" ]]; then stop_docker_machine_fail @@ -128,15 +139,11 @@ set_xauthority # can use global XAUTHORITY # start X server and set DISPLAY -if [[ "$OS" =~ "windows" ]]; then +if [[ "$OS" == "windows" ]] || [[ "$OS" == "wsl" ]]; then check_vcxsrv_running_print if [[ $? -ne 0 ]]; then find_command_suggested_path "vcxsrv" "/c/Program Files/VcXsrv" - if [[ $? -eq 0 ]]; then - # duplicate the functionality of the 'basename' command - # (remove the part between the last '/' and the end of line '$') - VCXSRV_DIR="$(echo ${vcxsrv_cmd}|sed 's/\/[^\/]\{1,\}$//')" - else + if [[ $? -ne 0 ]]; then echo "Could not find vcxsrv command. Please run XLaunch manually" | tee -a "$LOGFILE" cleanup 1 fi @@ -145,10 +152,18 @@ if [[ "$OS" =~ "windows" ]]; then set_local_display_from_port 0 # DISPLAY is set - find_command_suggested_path "xauth" "/c/Program Files/VcXsrv" - if [[ $? -ne 0 ]]; then - echo "Could not find xauth command. Please check VcXsrv installation" | tee -a "$LOGFILE" - cleanup 1 + if [[ "$OS" == "windows" ]]; then + find_command_suggested_path "xauth" "/c/Program Files/VcXsrv" + if [[ $? -ne 0 ]]; then + echo "Could not find xauth command. Please check VcXsrv installation" | tee -a "$LOGFILE" + cleanup 1 + fi + else + find_program_print xauth + if [[ $? -ne 0 ]]; then + echo "Could not find xauth command." | tee -a "$LOGFILE" + cleanup 1 + fi fi set_local_display_from_port 0 @@ -213,7 +228,7 @@ fi # ignore hostname in Xauthority by setting FamilyWild mask # https://stackoverflow.com/questions/16296753/can-you-run-gui-applications-in-a-docker-container/25280523#25280523 -KEYS_TO_CONVERT=$(echo "$(get_xauth_keys)" | grep -v '^ffff') # we know get_xauth_keys will succeed after above +KEYS_TO_CONVERT=$(echo "$(get_host_xauth_keys)" | grep -v '^ffff') # we know get_xauth_keys will succeed after above if [[ -n $KEYS_TO_CONVERT ]]; then print_header_message "Updating xauth keys for use with docker... " # this command is too complicated to try to wrap in a function... quoting all variables @@ -238,7 +253,7 @@ fi check_hnn_out_perms_host if [[ $? -ne "0" ]]; then - create_hnn_out_fail + fix_hnn_out_perms_host_fail fi echo | tee -a "$LOGFILE" @@ -246,7 +261,7 @@ echo "Starting HNN" | tee -a "$LOGFILE" echo "--------------------------------------" | tee -a "$LOGFILE" # check if container exists -output_existing_container_command > /dev/null +output_existing_container_command &> /dev/null if [[ $? -eq "0" ]]; then # container does exist, so check if it is running print_header_message "Checking for running HNN container... " @@ -257,6 +272,23 @@ if [[ $? -eq "0" ]]; then else ALREADY_RUNNING=0 echo "not found" | tee -a "$LOGFILE" + print_header_message "Starting existing HNN container... " + start_container_silent + if [[ $? -eq 0 ]]; then + echo "done" | tee -a "$LOGFILE" + else + echo "*failed*" | tee -a "$LOGFILE" + if [[ $TRAVIS_TESTING -eq 1 ]]; then + find_existing_container_print + if [[ $? -eq "0" ]]; then + print_header_message "Removing old container... " + remove_container_fail + fi + else + echo "Removing old container might resolve failure. Verification required." | tee -a "$LOGFILE" + prompt_remove_container_fail + fi + fi fi else ALREADY_RUNNING=0 @@ -264,30 +296,33 @@ fi if [[ $ALREADY_RUNNING -eq 0 ]]; then # start the container - docker_run_print - if [[ $? -ne 0 ]]; then - RETRY=1 - # try starting again - docker_run_print - if [[ $? -ne 0 ]]; then - cleanup 2 - fi - fi + docker_run_fail # container is up - # Workaround because in the qemu docker-machine (used with mac and windows). - # volume mounts don't work (hnn_out and hnn_source_code). - if [[ $TRAVIS_TESTING -eq 1 ]] && [[ ! "$OS" =~ "linux" ]]; then + if [[ $USE_SSH -eq 1 ]]; then + copy_ssh_files_to_container_fail + fi + + if [[ $TRAVIS_TESTING -eq 1 ]]; then + # Use Travis's checkout of the code inside the container copy_hnn_source_fail + copy_hnn_out_fail fi fi -if [[ $TRAVIS_TESTING -eq 0 ]] || ([[ ! "$OS" =~ "windows" ]] && [[ ! "$OS" =~ "wsl" ]]); then +if [[ $TRAVIS_TESTING -eq 1 ]]; then + # we may not have write access if the user changed (i.e. now USE_SSH=1) + change_hnn_source_perms_fail +fi + +fix_hnn_out_perms_container_print +if [[ $? -ne 0 ]]; then + fix_hnn_out_perms_host_fail +fi + +if [[ "$CONTAINER_TYPE" == "linux" ]]; then check_x_port_container_fail # could open port - - check_hnn_out_perms_container_fail - # can read/write to hnn_out directory fi setup_xauthority_in_container_fail @@ -302,9 +337,9 @@ if [[ $USE_SSH -eq 0 ]]; then cleanup 2 fi else - if [[ $TRAVIS_TESTING -eq 1 ]] && ([[ "$OS" =~ "windows" ]] || [[ "$OS" == "wsl" ]]); then + if [[ "$CONTAINER_TYPE" == "windows" ]]; then echo "SSH mode not supported with windows containers" - cleanup 2 + cleanup 1 fi if [[ $ALREADY_RUNNING -eq 0 ]]; then # need to start sshd @@ -323,7 +358,7 @@ else ssh_start_hnn_print if [[ $? -ne 0 ]]; then # failure could be caused by bad ssh keys - copy_ssh_files_to_running_container_fail + copy_ssh_files_to_container_fail RETRY=1 DEBUG=1 @@ -336,7 +371,7 @@ else fi if [[ $TRAVIS_TESTING -eq 1 ]]; then - echo "Testing mode: "/home/$TEST_USER/hnn/hnn_docker.log"" | tee -a "$LOGFILE" + echo "Testing mode: $LOGFILE" | tee -a "$LOGFILE" fi ## DONE! all successful cases reach here diff --git a/installer/centos/centos6-installer.sh b/installer/centos/hnn-centos6.sh similarity index 100% rename from installer/centos/centos6-installer.sh rename to installer/centos/hnn-centos6.sh diff --git a/installer/centos/centos7-installer.sh b/installer/centos/hnn-centos7.sh similarity index 100% rename from installer/centos/centos7-installer.sh rename to installer/centos/hnn-centos7.sh diff --git a/installer/centos/native_install.md b/installer/centos/native_install.md index 991a895e0..00cd5dd2a 100644 --- a/installer/centos/native_install.md +++ b/installer/centos/native_install.md @@ -5,15 +5,15 @@ The script below assumes that it can update OS packages for python and prerequis - CentOS 7: [centos7-installer.sh](centos7-installer.sh) ```bash - chmod +x ./centos7-installer.sh - ./centos7-installer.sh + chmod +x ./hnn-centos7.sh + ./hnn-centos7.sh ``` - CentOS 6 (no longer maintained): [centos6-installer.sh](centos6-installer.sh) ```bash - chmod +x ./centos6-installer.sh - ./centos6-installer.sh + chmod +x ./hnn-centos6.sh + ./hnn-centos6.sh ``` ## Troubleshooting diff --git a/installer/docker/check_hnn_out_perms.sh b/installer/docker/check_hnn_out_perms.sh index f2ae909ea..47620daa5 100755 --- a/installer/docker/check_hnn_out_perms.sh +++ b/installer/docker/check_hnn_out_perms.sh @@ -13,11 +13,11 @@ for dir in "${SYSTEM_USER_DIR}/hnn_out" "${SYSTEM_USER_DIR}/hnn_out/data" "${SYS mkdir "$dir" fi if [[ $? -ne 0 ]]; then - echo "failed" - echo "Error: $dir has incorrect permissions or could not create $dir. Can't start HNN." + echo "*failed*" + echo "Error: $dir has incorrect permissions or could not create $dir. Output from ls -l:" ls -ld "$dir" exit 1 fi done -echo "ok" +echo "done" diff --git a/installer/ubuntu/hnn-ubuntu.sh b/installer/ubuntu/hnn-ubuntu.sh new file mode 100755 index 000000000..65e7dc8ce --- /dev/null +++ b/installer/ubuntu/hnn-ubuntu.sh @@ -0,0 +1,220 @@ +#!/bin/bash + +set -e + +[[ ${NEURON_VERSION} ]] || NEURON_VERSION=7.7 +[[ "$LOGFILE" ]] || LOGFILE="ubuntu_install.log" + +function start_download { + echo "Downloading $2" + let __retries=5 + while [[ $__retries -gt 0 ]]; do + curl -Lo "$1" --retry 5 --retry-delay 30 "$2" && break + (( __retries-- )) + done + if [[ $__retries -eq 0 ]]; then + echo "Error: failed to download $2." + exit 1 + fi +} + +function wait_for_pid { + echo -n "Waiting for PID $1... " + wait $1 && { + echo "done" + echo "Finished $2" + } || { + echo "*failed*" + echo "Error: failed $2" + exit 1 + } +} + +function script_fail { + echo -ne "\n******* install script failed. output from log below *******\n" + cat "$LOGFILE" + exit 2 +} + +trap script_fail EXIT + +DISTRIB=$(grep DISTRIB_CODENAME /etc/lsb-release | cut -d'=' -f2) +echo "Starting installation of HNN on dectected Ubuntu OS: $DISTRIB" | tee -a "$LOGFILE" +echo "Output in log file: $LOGFILE" + +[[ $PYTHON_VERSION ]] || { + if [[ "$DISTRIB" =~ "xenial" ]]; then + PYTHON_VERSION=3.5 + elif [[ "$DISTRIB" =~ "bionic" ]]; then + PYTHON_VERSION=3.6 + elif [[ "$DISTRIB" =~ "disco" ]]; then + PYTHON_VERSION=3.7 + else + echo "Error: Ubuntu distribtion $DISTRIB not supported" | tee -a "$LOGFILE" + exit 1 + fi +} +echo "Using python version $PYTHON_VERSION" | tee -a "$LOGFILE" + +# avoid questions from debconf +export DEBIAN_FRONTEND=noninteractive + +echo "Updating package repository..." | tee -a "$LOGFILE" +sudo -E apt-get update &> "$LOGFILE" + +echo "Downloading Python packages..." | tee -a "$LOGFILE" +apt-get download \ + make gcc g++ python3 python3-pip python3-dev python3-tk python3-setuptools \ + libncurses5 libreadline5 libdbus-1-3 libopenmpi-dev &> "$LOGFILE" & +APT_DOWNLOAD=$! + +echo "Waiting for package download to finish..." +NAME="downloading python packages" +wait_for_pid "${APT_DOWNLOAD}" "$NAME" + +echo "Downloading NEURON..." +URL="https://neuron.yale.edu/ftp/neuron/versions/v${NEURON_VERSION}/nrn-${NEURON_VERSION}.$(uname -p)-linux.deb" +FILENAME="/tmp/nrn.deb" +start_download "$FILENAME" "$URL" > /dev/null & +NEURON_DOWNLOAD=$! + +echo "Updating OS python packages..." | tee -a "$LOGFILE" +if [[ "${PYTHON_VERSION}" =~ "3.7" ]] && [[ "$DISTRIB" =~ "bionic" ]]; then + sudo -E apt-get install --no-install-recommends -y python3.7 python3-pip python3.7-tk python3.7-dev &> "$LOGFILE" && \ + sudo python3.7 -m pip install --upgrade pip setuptools &> "$LOGFILE" + sudo ln -s /usr/lib/python3/dist-packages/apt_pkg.cpython-36m-x86_64-linux-gnu.so \ + /usr/lib/python3/dist-packages/apt_pkg.so +else + sudo -E apt-get install --no-install-recommends -y python3 python3-pip python3-tk python3-setuptools &> "$LOGFILE" && \ + sudo pip3 install --upgrade pip &> "$LOGFILE" +fi + +if which python3 &> /dev/null; then + export PYTHON=$(which python3) + if pip3 -V &> /dev/null; then + PIP=pip3 + else + PIP=pip + fi +elif which python &> /dev/null; then + export PYTHON=$(which python) + PIP=pip +fi +echo "Using python: $PYTHON" | tee -a "$LOGFILE" + +echo "Downloading python packages for HNN with pip..." | tee -a "$LOGFILE" +$PIP download matplotlib PyOpenGL \ + pyqt5 pyqtgraph scipy numpy nlopt psutil &> "$LOGFILE" & +PIP_PID=$! + +echo "Installing OS compilation toolchain..." | tee -a "$LOGFILE" +# get prerequisites from pip. requires gcc to build psutil +sudo -E apt-get install --no-install-recommends -y \ + make gcc g++ python3-dev &> "$LOGFILE" + +echo "Waiting for python packages for HNN downloads to finish..." +NAME="downloading python packages for HNN " +wait_for_pid "${PIP_PID}" "$NAME" + +echo "Installing python packages for HNN with pip..." | tee -a "$LOGFILE" +$PIP install --no-cache-dir --user matplotlib PyOpenGL \ + pyqt5 pyqtgraph scipy numpy nlopt psutil &> "$LOGFILE" + +echo "Waiting for NEURON download to finish..." +NAME="downloading NEURON package" +wait_for_pid "${NEURON_DOWNLOAD}" "$NAME" + +echo "Downloading runtime prerequisite packages..." | tee -a "$LOGFILE" +apt-get download \ + openmpi-bin lsof libfontconfig1 libxext6 libx11-xcb1 libxcb-glx0 \ + libxkbcommon-x11-0 libgl1-mesa-glx \ + libc6-dev libtinfo-dev libncurses5-dev \ + libx11-dev libreadline-dev &> "$LOGFILE" & +APT_DOWNLOAD=$! + +# Install NEURON +echo "Installing NEURON $NEURON_VERSION precompiled package..." | tee -a "$LOGFILE" +(sudo -E dpkg -i /tmp/nrn.deb &> "$LOGFILE" && \ + rm -f /tmp/nrn.deb &> "$LOGFILE" ) & +NEURON_PID=$! + +# save dir installing hnn to +startdir=$(pwd) + +if [[ $TRAVIS_TESTING -ne 1 ]]; then + # setup HNN itself + source_code_dir="$startdir/hnn_source_code" + if [ -d "$source_code_dir" ]; then + echo "Updating HNN source code..." | tee -a "$LOGFILE" + + cd hnn_source_code + if [ -d "$source_code_dir/.git" ]; then + git pull origin master &> "$LOGFILE" + fi + else + echo "Cloning HNN..." | tee -a "$LOGFILE" + git clone https://github.com/jonescompneurolab/hnn.git hnn_source_code &> "$LOGFILE" && \ + cd hnn_source_code + fi +else + source_code_dir="$startdir" + if [[ ! -f hnn.py ]]; then + echo "Couldn't find HNN source code at $startdir" | tee -a "$LOGFILE" + exit 1 + fi +fi + +echo "Waiting for NEURON install to finish..." +NAME="installing NEURON" +wait_for_pid "${NEURON_PID}" "$NAME" + +if [[ $TRAVIS_TESTING -ne 1 ]]; then + echo "Building HNN..." | tee -a "$LOGFILE" + make -j4 &> "$LOGFILE" + MAKE_PID=$! +fi + +# create the global session variables +echo '# these lines define global session variables for HNN' >> ~/.bashrc +echo "export PATH=\$PATH:\"$source_code_dir\"" >> ~/.bashrc +if [[ "${PYTHON_VERSION}" =~ "3.6" ]] && [[ "${PYTHON_VERSION}" =~ "3.7" ]]; then + echo "export PYTHONPATH=/usr/local/nrn/lib/python" >> ~/.bashrc +fi + +if [[ -d "$HOME/Desktop" ]]; then + { + mkdir -p "$HOME/.local/share/icons" && \ + cp -f hnn.png "$HOME/.local/share/icons/" && \ + cp -f hnn.desktop "$HOME/Desktop" && \ + sed -i "s~/home/hnn_user\(.*\)$~\"$startdir\"\1~g" "$HOME/Desktop/hnn.desktop" && \ + chmod +x "$HOME/Desktop/hnn.desktop" + } &> "$LOGFILE" +fi + +echo "Waiting for prerequisite package downloads to finish..." +NAME="downloading prerequisite packages" +wait_for_pid "${APT_DOWNLOAD}" "$NAME" + +echo "Installing prerequisites..." | tee -a "$LOGFILE" +sudo -E apt-get install --no-install-recommends -y \ + openmpi-bin lsof \ + libfontconfig1 libxext6 libx11-xcb1 libxcb-glx0 \ + libxkbcommon-x11-0 libgl1-mesa-glx \ + libncurses5 libreadline5 libdbus-1-3 libopenmpi-dev \ + libc6-dev libtinfo-dev libncurses5-dev \ + libx11-dev libreadline-dev &> "$LOGFILE" + +# Clean up a little +echo "Cleaning up..." | tee -a "$LOGFILE" +sudo -E apt-get clean &> "$LOGFILE" + +if [[ $TRAVIS_TESTING -ne 1 ]]; then + echo "Waiting for HNN module build to finish..." + NAME="building HNN modules" + wait_for_pid "${MAKE_PID}" "$NAME" +fi + +echo "HNN installation successful" | tee -a "$LOGFILE" +echo "Source code is at $source_code_dir" | tee -a "$LOGFILE" + +trap EXIT \ No newline at end of file diff --git a/installer/ubuntu/installer.sh b/installer/ubuntu/installer.sh deleted file mode 100755 index 1432c6200..000000000 --- a/installer/ubuntu/installer.sh +++ /dev/null @@ -1,128 +0,0 @@ -#!/bin/bash - -set -e - -[[ ${NEURON_VERSION} ]] || NEURON_VERSION=7.7 -[[ "$LOGFILE" ]] || LOGFILE="ubuntu_install.log" - -DISTRIB=$(grep DISTRIB_CODENAME /etc/lsb-release | cut -d'=' -f2) -echo "Starting installation of HNN on dectected Ubuntu OS: $DISTRIB" | tee -a "$LOGFILE" -echo "Output in log file: $LOGFILE" | tee -a "$LOGFILE" - -if [[ "$DISTRIB" =~ "xenial" ]]; then - PYTHON_VERSION=3.5 -elif [[ "$DISTRIB" =~ "bionic" ]]; then - PYTHON_VERSION=3.6 -elif [[ "$DISTRIB" =~ "disco" ]]; then - PYTHON_VERSION=3.7 -else - echo "Error: Ubuntu distribtion $DISTRIB not supported" | tee -a "$LOGFILE" - exit 1 -fi -echo "Will install python version: $PYTHON_VERSION" | tee -a "$LOGFILE" - - -# avoid questions from debconf -export DEBIAN_FRONTEND=noninteractive - -echo "Updating OS python packages..." | tee -a "$LOGFILE" -sudo apt-get update &> "$LOGFILE" -if [[ "${PYTHON_VERSION}" =~ "3.7" ]] && [[ "$DISTRIB" =~ "bionic" ]]; then - sudo apt-get install --no-install-recommends -y python3.7 python3-pip python3.7-tk python3.7-dev &> "$LOGFILE" && \ - sudo python3.7 -m pip install --upgrade pip setuptools &> "$LOGFILE" -else - sudo apt-get install --no-install-recommends -y python3 python3-pip python3-tk python3-setuptools &> "$LOGFILE" && \ - sudo pip3 install --upgrade pip &> "$LOGFILE" -fi - -echo "Installing OS compilation toolchain..." | tee -a "$LOGFILE" -# get prerequisites from pip. requires gcc to build psutil -sudo apt-get install --no-install-recommends -y \ - make gcc g++ python3-dev &> "$LOGFILE" - -which python > /dev/null 2>&1 -if [[ $? -eq 0 ]]; then - # there is another python executable that might be version 2.7 - export PYTHON=$(which python3) - PIP=pip3 -else - PIP=pip -fi -echo "Using python: $PYTHON" | tee -a "$LOGFILE" - -echo "Installing python pacakges for HNN with pip..." | tee -a "$LOGFILE" -$PIP install --no-cache-dir --user matplotlib PyOpenGL \ - pyqt5 pyqtgraph scipy numpy nlopt psutil &> "$LOGFILE" - -# base prerequisites packages -echo "Installing OS openmpi..." | tee -a "$LOGFILE" -sudo apt-get install --no-install-recommends -y \ - openmpi-bin lsof &> "$LOGFILE" - -# Qt prerequisites packages -echo "Installing Qt OS prerequisites..." | tee -a "$LOGFILE" -sudo apt-get install --no-install-recommends -y \ - libfontconfig libxext6 libx11-xcb1 libxcb-glx0 \ - libxkbcommon-x11-0 &> "$LOGFILE" - -echo "Installing NEURON prerequisites..." | tee -a "$LOGFILE" -# NEURON runtime prerequisites -sudo apt-get install --no-install-recommends -y \ - libncurses5 libreadline5 libdbus-1-3 libopenmpi-dev &> "$LOGFILE" - -# Install NEURON -echo "Installing NEURON $NEURON_VERSION precompiled package..." | tee -a "$LOGFILE" -wget -q https://neuron.yale.edu/ftp/neuron/versions/v${NEURON_VERSION}/nrn-${NEURON_VERSION}.$(uname -p)-linux.deb -O /tmp/nrn.deb &> "$LOGFILE" && \ - sudo dpkg -i /tmp/nrn.deb &> "$LOGFILE" && \ - rm -f /tmp/nrn.deb &> "$LOGFILE" - -# HNN build prerequisites -echo "Installing HNN build prerequisites..." | tee -a "$LOGFILE" -sudo apt-get install --no-install-recommends -y \ - libc6-dev libtinfo-dev libncurses-dev \ - libx11-dev libreadline-dev &> "$LOGFILE" - -# save dir installing hnn to -startdir=$(pwd) - -if [[ $TRAVIS_TESTING -ne 1 ]]; then - # setup HNN itself - if [ -d "$startdir/hnn_source_code" ]; then - echo "Updating HNN source code..." | tee -a "$LOGFILE" - - cd hnn_source_code - if [ -d "$startdir/hnn_source_code/.git" ]; then - git pull origin master &> "$LOGFILE" - fi - else - echo "Cloning HNN..." | tee -a "$LOGFILE" - git clone https://github.com/jonescompneurolab/hnn.git hnn_source_code &> "$LOGFILE" && \ - cd hnn_source_code - fi - - echo "Building HNN..." | tee -a "$LOGFILE" - make -j4 &> "$LOGFILE" - - cd "$startdir" -fi - -# Clean up a little -echo "Cleaning up..." | tee -a "$LOGFILE" -sudo apt-get clean &> "$LOGFILE" - -# create the global session variables -echo '# these lines define global session variables for HNN' >> ~/.bashrc -echo "export PATH=\$PATH:\"$startdir/hnn_source_code\"" >> ~/.bashrc - -if [[ -d "$HOME/Desktop" ]]; then - { - mkdir -p "$HOME/.local/share/icons" && \ - cp -f hnn.png "$HOME/.local/share/icons/" && \ - cp -f hnn.desktop "$HOME/Desktop" && \ - sed -i "s~/home/hnn_user\(.*\)$~\"$startdir\"\1~g" "$HOME/Desktop/hnn.desktop" && \ - chmod +x "$HOME/Desktop/hnn.desktop" - } &> "$LOGFILE" -fi - -echo "HNN installation successful" | tee -a "$LOGFILE" -echo "Source code is at $startdir/hnn_source_code" | tee -a "$LOGFILE" diff --git a/installer/windows/Dockerfile b/installer/windows/Dockerfile index 50cdf1c5b..8cfd878fb 100644 --- a/installer/windows/Dockerfile +++ b/installer/windows/Dockerfile @@ -1,4 +1,4 @@ -FROM mcr.microsoft.com/windows/servercore:ltsc2019 +FROM mcr.microsoft.com/windows/servercore:1909 # SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue';"] SHELL ["cmd", "/S", "/C"] @@ -28,8 +28,8 @@ COPY installer/docker/hnn_envs /home/hnn_user COPY installer/docker/start_hnn.sh /home/hnn_user COPY installer/docker/check_hnn_out_perms.sh /home/hnn_user -ADD installer/windows/hnn.ps1 /home/hnn_user/Downloads/ -RUN powershell -executionpolicy bypass -File "C:\home\hnn_user\Downloads\hnn.ps1" && \ +ADD installer/windows/hnn-windows.ps1 /home/hnn_user/Downloads/ +RUN powershell -executionpolicy bypass -File "C:\home\hnn_user\Downloads\hnn-windows.ps1" && \ powershell -Command "$env:PATH = 'C:\home\hnn_user\AppData\Local\Programs\Git\bin;' + $env:PATH + \ ';"C:\Program Files\VcXsrv";C:\home\hnn_user\Miniconda3\Scripts;C:\home\hnn_user\Miniconda3\envs\hnn;' + \ 'C:\home\hnn_user\Miniconda3\envs\hnn\Scripts;C:\home\hnn_user\Miniconda3\envs\hnn\Library\bin' ; \ @@ -38,7 +38,8 @@ RUN powershell -executionpolicy bypass -File "C:\home\hnn_user\Downloads\hnn.ps1 del home\hnn_user\Downloads\*.* /Q RUN cd home\hnn_user && \ - move /y hnn hnn_source_code + move /y hnn hnn_source_code && \ + mkdir .ssh WORKDIR /home/hnn_user/hnn_source_code diff --git a/installer/windows/hnn.ps1 b/installer/windows/hnn-windows.ps1 similarity index 100% rename from installer/windows/hnn.ps1 rename to installer/windows/hnn-windows.ps1 diff --git a/installer/windows/native_install.md b/installer/windows/native_install.md index 4ad25aa0b..20ecd9383 100644 --- a/installer/windows/native_install.md +++ b/installer/windows/native_install.md @@ -2,7 +2,7 @@ This method will run HNN without using virtualization, meaning the GUI may feel more responsive and simulations may run slightly faster. However, there is a possibility that differences in the base environment may require additional troubleshooting. Thus, it is best suited for advanced users, and we recommend the [Docker-based install instructions](README.md). -With this method, a [PowerShell install script](hnn.ps1) will manage downloading all prerequisites, including Python (Anaconda), NEURON, and Git for Windows. +With this method, a [PowerShell install script](hnn-windows.ps1) will manage downloading all prerequisites, including Python (Anaconda), NEURON, and Git for Windows. ## Requirements @@ -20,19 +20,19 @@ The PowerShell script used below will create a new directory called "hnn" in the ```powershell cd hnn - powershell.exe -ExecutionPolicy Bypass -File .\installer\windows\hnn.ps1 + powershell.exe -ExecutionPolicy Bypass -File .\installer\windows\hnn-windows.ps1 ``` OR to download and run the script from a url: ```powershell - @"%SystemRoot%\System32\WindowsPowerShell\v1.0\powershell.exe" -NoProfile -InputFormat None -ExecutionPolicy Bypass -Command "iex ((New-Object System.Net.WebClient).DownloadString('https://raw.githubusercontent.com/jonescompneurolab/hnn/master/installer/windows/hnn.ps1'))" + @"%SystemRoot%\System32\WindowsPowerShell\v1.0\powershell.exe" -NoProfile -InputFormat None -ExecutionPolicy Bypass -Command "iex ((New-Object System.Net.WebClient).DownloadString('https://raw.githubusercontent.com/jonescompneurolab/hnn/master/installer/windows/hnn-windows.ps1'))" ``` OR from a powershell prompt: ```powershell - Set-ExecutionPolicy Bypass -Scope Process -Force; iex ((New-Object System.Net.WebClient).DownloadString('https://raw.githubusercontent.com/jonescompneurolab/hnn/master/installer/windows/hnn.ps1')) + Set-ExecutionPolicy Bypass -Scope Process -Force; iex ((New-Object System.Net.WebClient).DownloadString('https://raw.githubusercontent.com/jonescompneurolab/hnn/master/installer/windows/hnn-windows.ps1')) ``` - There will be a permission prompt to install Microsoft MPI and a couple of terminal windows will diff --git a/scripts/docker_functions.sh b/scripts/docker_functions.sh index 5fee2cdfa..55f68c812 100644 --- a/scripts/docker_functions.sh +++ b/scripts/docker_functions.sh @@ -38,6 +38,45 @@ function check_var { fi } +function get_wsl_home_path { + # no arguments + check_var ESC_STR + + local __command + local __index + local __arg + local __send_args + local __home + + log_header_message "Getting path outside of WSL... " + __command=("powershell.exe" "\$env:HOME" "tr" "-d '\r'") + let __index=0 + for __arg in "${__command[@]}"; do + __send_args[$__index]=$(echo $__arg|sed "s/ /$ESC_STR/g") + (( __index++ )) + done + __home=$(output_run_piped_command "${__send_args[@]}") + if [[ $? -ne 0 ]]; then + echo "*failed*" >> "$LOGFILE" + return 2 + else + echo "done" >> "$LOGFILE" + fi + + log_header_message "Turning into UNIX path... " + COMMAND=(3 "cygpath.exe" "-u" "$__home") + convert_COMMAND_to_escaped_array + __home=$(output_run_command_arguments "${ESCAPED_COMMAND[@]}") + if [[ $? -ne 0 ]]; then + echo "*failed*" >> "$LOGFILE" + return 2 + else + echo "done" >> "$LOGFILE" + fi + + echo "$__home" +} + function get_os { OS_OUTPUT=$(uname -a) if [[ $OS_OUTPUT =~ "MINGW" ]] || [[ $OS_OUTPUT =~ "MSYS" ]]; then @@ -68,34 +107,38 @@ function set_globals { SSHD_STARTED=0 NEW_XAUTH_KEYS=0 ESC_STR="%^%" + VCXSRV_PID= + + SSH_PRIVKEY="$(pwd)/installer/docker/id_rsa_hnn" + SSH_PUBKEY="$(pwd)/installer/docker/id_rsa_hnn.pub" + SSH_AUTHKEYS="$(pwd)/installer/docker/authorized_keys" + + export USE_SSH UPGRADE STOP START RETRY UNINSTALL HNN_DOCKER_IMAGE HNN_CONTAINER_NAME + export SYSTEM_USER_DIR ALREADY_RUNNING SSHD_STARTED NEW_XAUTH_KEYS ESC_STR VCXSRV_PID + OS=$(get_os) __command_status=$? if [[ $__command_status -ne 0 ]]; then echo "Failed to get OS type" | tee -a "$LOGFILE" - exit $__command_status + return $__command_status fi if [[ "$OS" == "wsl" ]]; then - echo "WSL is not supported yet" - # exit 1 + export SYSTEM_USER_DIR="$(get_wsl_home_path)" + if [[ $? -ne 0 ]]; then + return $? + fi set_local_display_from_port 0 fi # if display hasn't been set yet, give it a default value for the OS [[ "$DISPLAY" ]] || set_local_display_from_port 0 - SSH_PRIVKEY="$(pwd)/installer/docker/id_rsa_hnn" - SSH_PUBKEY="$(pwd)/installer/docker/id_rsa_hnn.pub" - SSH_AUTHKEYS="$(pwd)/installer/docker/authorized_keys" - if [[ "$OS" == "mac" ]]; then function timeout { perl -e 'alarm shift; exec @ARGV' "$@" } export timeout fi - - export USE_SSH UPGRADE STOP START RETRY UNINSTALL HNN_DOCKER_IMAGE HNN_CONTAINER_NAME - export SYSTEM_USER_DIR ALREADY_RUNNING SSHD_STARTED NEW_XAUTH_KEYS ESC_STR } function errexit() { @@ -134,10 +177,10 @@ function cleanup { echo "cleanup() called from: ${FUNCNAME[1]} (L:${BASH_LINENO[0]})" >> "$LOGFILE" if [ ! -z "${VCXSRV_PID}" ]; then stop_vcxsrv - # if [[ $? -ne 0 ]]; then - echo "Killing VcXsrv PID ${VCXSRV_PID}" >> "$LOGFILE" - kill ${VCXSRV_PID} &> /dev/null - # fi + if [[ $? -ne 0 ]]; then + echo "Killing VcXsrv PID ${VCXSRV_PID}" >> "$LOGFILE" + kill ${VCXSRV_PID} &> /dev/null + fi fi if [[ $__failed -eq "0" ]]; then @@ -226,7 +269,7 @@ function output_run_piped_command { echo -e "\n ** Command: $__binary1 $__command_args1 | $__binary2 $__command_args2" >> "$LOGFILE" echo -n " ** Stderr: " >> "$LOGFILE" - __output=$("$__binary1" $__command_args1 | "$__binary2" $__command_args2 2>> "$LOGFILE") + __output=$("$__binary1" $__command_args1 2>> "$LOGFILE" | "$__binary2" $__command_args2 2>> "$LOGFILE") __command_status=$? if [[ ! -z "$__output" ]]; then echo -e "\n ** Stdout: $__output" | tr -d '\r' >> "$LOGFILE" @@ -265,8 +308,8 @@ function output_run_command_arguments { # note that output here is not expected to go to stdout, so redirect to LOGFILE re='^[0-9]+$' if ! [[ $1 =~ $re ]] ; then - echo -e "\nError: $FUNCNAME argument $1 is not a number" >> "$LOGFILE" - cleanup 1 + echo -e "\nError: $FUNCNAME: argument '$1' is not a number" >> "$LOGFILE" + exit 1 fi check_var LOGFILE @@ -281,7 +324,7 @@ function output_run_command_arguments { if [[ $__num_args -ne $__expected_num_args ]]; then echo -e "\nError: $FUNCNAME (L:${BASH_LINENO[0]}) must have $__expected_num_args arguments" >> "$LOGFILE" echo "Got $__num_args arguments: $@" >> "$LOGFILE" - cleanup 1 + exit 1 fi local __index @@ -322,11 +365,16 @@ function output_run_command_arguments { __command_status=$? if [[ -n "$__output" ]]; then echo >> "$LOGFILE" - echo " ** Stdout: $__output" | tr -d '\r' >> "$LOGFILE" + if [[ $SUPPRESS_OUTPUT -eq 1 ]]; then + echo " ** Stdout: ** suppresed **" >> "$LOGFILE" + SUPPRESS_OUTPUT=1 + else + echo " ** Stdout: $__output" | tr -d '\r' >> "$LOGFILE" + fi fi # send output back to caller - echo "$__output" + echo "$__output" | tr -d '\r' if [[ $__command_status -eq 0 ]]; then true @@ -420,7 +468,16 @@ function remove_container_fail { check_var docker_cmd check_var HNN_CONTAINER_NAME - run_command_print_status_failure_exit "$docker_cmd rm -fv $HNN_CONTAINER_NAME" + COMMAND=(4 "$docker_cmd" "rm" "-fv" "$HNN_CONTAINER_NAME") + convert_COMMAND_to_escaped_array + output_run_command_arguments "${ESCAPED_COMMAND[@]}" &> /dev/null + if [[ $? -ne "0" ]]; then + echo "*failed*" | tee -a "$LOGFILE" + echo -e "\nFailure to remove container can sometimes be fixed by restarting Docker" + cleanup 2 + else + echo "done" | tee -a "$LOGFILE" + fi } function print_header_message { @@ -442,20 +499,42 @@ function print_header_message_short { echo -n "$1" | tee -a "$LOGFILE" } +function log_header_message { + check_args "$@" $# 1 + check_var LOGFILE + + # first arg is message to print. surrounded by newlines in "$LOGFILE" + echo >> "$LOGFILE" + echo -n "$1" >> "$LOGFILE" + echo >> "$LOGFILE" +} + function start_vcxsrv_print { check_var LOGFILE - check_var VCXSRV_DIR + check_var vcxsrv_cmd - print_header_message "Starting VcXsrv... " - echo -e "\n ** Command: ${VCXSRV_DIR}/vcxsrv.exe -wgl -multiwindow 2>&1 &" >> "$LOGFILE" - if [[ $DEBUG -eq 1 ]] || [[ $VERBOSE -eq 1 ]]; then - echo -n " ** Output: " >> "$LOGFILE" - "${VCXSRV_DIR}/vcxsrv.exe" -wgl -multiwindow >> "$LOGFILE" 2>&1 & - else - "${VCXSRV_DIR}/vcxsrv.exe" -wgl -multiwindow > /dev/null 2>&1 & + local __vcxsrv_dir + + # duplicate the functionality of the 'basename' command + # (remove the part between the last '/' and the end of line '$') + __vcxsrv_dir="$(echo ${vcxsrv_cmd}|sed 's/\/[^\/]\{1,\}$//')" + if [[ ! -d "$__vcxsrv_dir" ]]; then + echo "Could not find directory for ${vcxsrv_cmd}" + cleanup 2 fi + + print_header_message "Starting VcXsrv... " + COMMAND=(3 "${__vcxsrv_dir}/vcxsrv.exe" "-wgl" "-multiwindow") + convert_COMMAND_to_escaped_array + output_run_command_arguments "${ESCAPED_COMMAND[@]}" &> /dev/null & VCXSRV_PID=$! - echo "done" | tee -a "$LOGFILE" + + log_header_message "Checking if VcXsrv is running..." + COMMAND=(3 "kill" "-0" "$VCXSRV_PID") + convert_COMMAND_to_escaped_array + output_run_command_arguments "${ESCAPED_COMMAND[@]}" &> /dev/null + fail_on_bad_exit $? + echo "Started VcXsrv with PID ${VCXSRV_PID}" >> "$LOGFILE" } @@ -465,7 +544,11 @@ function stop_vcxsrv { print_header_message "Stopping VcXsrv... " echo >> "$LOGFILE" - run_command_print_status "cmd.exe //c taskkill //F //IM vcxsrv.exe" + if [[ "$OS" == "wsl" ]]; then + run_command_print_status "cmd.exe /c taskkill /F /IM vcxsrv.exe" + else + run_command_print_status "cmd.exe //c taskkill //F //IM vcxsrv.exe" + fi if [[ $? -eq "0" ]]; then VCXSRV_PID= else @@ -516,6 +599,7 @@ function get_container_port { check_var LOGFILE check_var HNN_CONTAINER_NAME check_var docker_cmd + check_var TRAVIS_TESTING local __index local __send_args @@ -529,6 +613,7 @@ function get_container_port { # this is hardcoded in docker-machine-driver-qemu __ssh_port="5000" else + log_header_message "Getting port that maps to docker container port 22... " __command=("$docker_cmd" "port $HNN_CONTAINER_NAME 22") let __index=0 for __arg in "${__command[@]}"; do @@ -538,6 +623,7 @@ function get_container_port { __port_string=$(output_run_specific_command "${__send_args[@]}") if [[ $? -ne "0" ]]; then + echo "*failed*" >> "$LOGFILE" echo "failed to run ${__command[@]}" >> "$LOGFILE" return 1 fi @@ -545,9 +631,11 @@ function get_container_port { __ssh_port=$(echo $__port_string| cut -d':' -f 2) re='^[0-9]+$' if ! [[ $__ssh_port =~ $re ]] ; then + echo "*failed*" >> "$LOGFILE" echo "failed to get a port number from \"$__ssh_port\"" >> "$LOGFILE" return 1 fi + echo "done" >> "$LOGFILE" fi echo $__ssh_port @@ -565,6 +653,8 @@ function ssh_start_hnn_print { check_var DEBUG check_var VERBOSE check_var LOGFILE + check_var CONTAINER_TYPE + check_var TRAVIS_TESTING local __verbose local __ssh_port @@ -592,9 +682,13 @@ function ssh_start_hnn_print { # since we assigned port 6000, we can be certain of this DISPLAY export DISPLAY=127.0.0.1:0 - export XAUTHORITY="/tmp/.Xauthority" export TRAVIS_TESTING - export SYSTEM_USER_DIR + + if [[ "$CONTAINER_TYPE" == "windows" ]]; then + export XAUTHORITY="/c/home/hnn_user/.Xauthority" + else + export XAUTHORITY="/tmp/.Xauthority" + fi if [[ $RETRY -eq 1 ]]; then echo -n "(retry) " | tee -a "$LOGFILE" @@ -612,6 +706,7 @@ function ssh_start_hnn_print { # server is running) and port 6000 in the container we are ssh'ing into. Other options are to avoid # warnings about hostkeychecking and to not prompt for a password if public key authentication fails. print_header_message "Starting HNN GUI... " + COMMAND=(23 "ssh" "-o" "SendEnv=DISPLAY" "-o" "SendEnv=XAUTHORITY" "-o" "SendEnv=SYSTEM_USER_DIR" "-o" "SendEnv=TRAVIS_TESTING" \ "-o" "PasswordAuthentication=no" "-o" "UserKnownHostsFile=/dev/null" "-o" "StrictHostKeyChecking=no" \ "$__verbose" "-i" "$SSH_PRIVKEY" "-R" "6000:127.0.0.1:6000" "hnn_user@$__docker_host_ip" "-p" "$__ssh_port") @@ -630,16 +725,18 @@ function start_hnn_print { # runs start_hnn.sh script directly in container using "docker exec" check_var docker_cmd check_var TRAVIS_TESTING + check_var SYSTEM_USER_DIR check_var HNN_CONTAINER_NAME check_var LOGFILE + check_var CONTAINER_TYPE + check_var UID local __display # set DISPLAY for OS + print_header_message "Setting DISPLAY variable for $CONTAINER_TYPE containers... " __display=$(get_display_for_gui) - if [[ $? -ne 0 ]]; then - cleanup $? - fi + fail_on_bad_exit $? if [[ $RETRY -eq 1 ]]; then echo -n "(retry) " | tee -a "$LOGFILE" @@ -647,16 +744,18 @@ function start_hnn_print { fi print_header_message "Starting HNN GUI... " - if [[ $TRAVIS_TESTING -eq 1 ]] && [[ "$OS" =~ "windows" ]]; then - COMMAND=(11 "$docker_cmd" "exec" "--env" "SYSTEM_USER_DIR=$HOME" "--env" "TRAVIS_TESTING=$TRAVIS_TESTING" \ - "--env" "DISPLAY=$__display" "$HNN_CONTAINER_NAME" "bash" "/c/home/hnn_user/start_hnn.sh") - elif [[ $TRAVIS_TESTING -eq 1 ]] && [[ "$OS" == "wsl" ]]; then - COMMAND=(11 "$docker_cmd" "exec" "--env" "SYSTEM_USER_DIR=/$HOME" "--env" "TRAVIS_TESTING=$TRAVIS_TESTING" \ - "--env" "DISPLAY=$__display" "$HNN_CONTAINER_NAME" "bash" "/c/home/hnn_user/start_hnn.sh") + if [[ "$CONTAINER_TYPE" == "windows" ]]; then + __container_user="hnn_user" + __start_script="/c/home/hnn_user/start_hnn.sh" else - COMMAND=(12 "$docker_cmd" "exec" "--env" "SYSTEM_USER_DIR=$HOME" "--env" "TRAVIS_TESTING=$TRAVIS_TESTING" \ - "--env" "DISPLAY=$__display" "-u" "$UID" "$HNN_CONTAINER_NAME" "/home/hnn_user/start_hnn.sh") + __container_user="$UID" + __start_script="/home/hnn_user/start_hnn.sh" fi + COMMAND=(13 "$docker_cmd" "exec" "--env" "TRAVIS_TESTING=$TRAVIS_TESTING" \ + "--env" "SYSTEM_USER_DIR=$SYSTEM_USER_DIR" "--env" "DISPLAY=$__display" \ + "-u" "$__container_user" "$HNN_CONTAINER_NAME" \ + "bash" "$__start_script") + convert_COMMAND_to_escaped_array MSYS_NO_PATHCONV=1 output_run_command_arguments "${ESCAPED_COMMAND[@]}" &> /dev/null if [[ $? -ne "0" ]]; then @@ -684,15 +783,30 @@ function check_sshd_proc { MSYS_NO_PATHCONV=1 run_command_print_status "$docker_cmd exec $HNN_CONTAINER_NAME pgrep sshd" } -function create_hnn_out_fail { +function fix_hnn_out_perms_host_fail { # no arguments check_var HOME + check_var TRAVIS_TESTING - print_header_message "Creating ${HOME}/hnn_out... " - COMMAND=(2 "mkdir" "${HOME}/hnn_out") - convert_COMMAND_to_escaped_array - output_run_command_arguments "${ESCAPED_COMMAND[@]}" &> /dev/null + if [[ ! -e "${SYSTEM_USER_DIR}/hnn_out/" ]]; then + print_header_message "Creating ${HOME}/hnn_out... " + COMMAND=(2 "mkdir" "${HOME}/hnn_out") + convert_COMMAND_to_escaped_array + output_run_command_arguments "${ESCAPED_COMMAND[@]}" &> /dev/null + fail_on_bad_exit $? + return + fi + + print_header_message "Updating hnn_out permissions on host... " + find "${SYSTEM_USER_DIR}/hnn_out" -type d -exec chmod 777 {} \; >> "$LOGFILE" 2>&1 && \ + find "${SYSTEM_USER_DIR}/hnn_out" -type f -exec chmod 666 {} \; >> "$LOGFILE" 2>&1 fail_on_bad_exit $? + + if [[ $TRAVIS_TESTING -eq 1 ]] && [[ "$OS" == "mac" ]]; then + COMMAND=(2 "touch" "${SYSTEM_USER_DIR}/hnn_out/THIS_DIRECTORY_IS_SHARED_BETWEEN_DOCKER_AND_YOUR_OS") + convert_COMMAND_to_escaped_array + output_run_command_arguments "${ESCAPED_COMMAND[@]}" &> /dev/null + fi } function check_hnn_out_perms_host { @@ -700,62 +814,77 @@ function check_hnn_out_perms_host { check_var LOGFILE check_var HOME - COMMAND=(3 "test" "-w" "${HOME}/hnn_out") - convert_COMMAND_to_escaped_array - output_run_command_arguments "${ESCAPED_COMMAND[@]}" &> /dev/null + installer/docker/check_hnn_out_perms.sh if [[ $? -ne 0 ]]; then return 1 fi } -function check_hnn_out_perms_container_fail { +function check_hnn_out_perms_container_print { # no arguments # runs script in container that checks permissions for hnn_out are okay check_var docker_cmd check_var HNN_CONTAINER_NAME - check_var TRAVIS_TESTING + check_var CONTAINER_TYPE check_var LOGFILE check_var RETRY + check_var SYSTEM_USER_DIR + + local __script_path + local __user + if [[ $RETRY -eq 1 ]]; then echo -n "(retry) " | tee -a "$LOGFILE" RETRY=0 fi - print_header_message "Checking permissions of ${HOME}/hnn_out in container... " - COMMAND=(6 "$docker_cmd" "exec" "--env" "SYSTEM_USER_DIR=$HOME" "$HNN_CONTAINER_NAME" "/home/hnn_user/check_hnn_out_perms.sh") + print_header_message "Checking permissions of ${SYSTEM_USER_DIR}/hnn_out in container... " + if [[ "$CONTAINER_TYPE" == "windows" ]]; then + __script_path="bash -c /c/home/hnn_user/check_hnn_out_perms.sh" + else + __script_path="/home/hnn_user/check_hnn_out_perms.sh" + fi + COMMAND=(6 "$docker_cmd" "exec" "$HNN_CONTAINER_NAME" "bash" "-c" "$__script_path") convert_COMMAND_to_escaped_array MSYS_NO_PATHCONV=1 output_run_command_arguments "${ESCAPED_COMMAND[@]}" &> /dev/null if [[ $? -ne 0 ]]; then echo "*failed*" | tee -a "$LOGFILE" - check_hnn_out_perms_host - if [[ $? -ne 0 ]]; then - if [[ "$OS" == "linux" ]]; then - print_header_message "Updating permissions of hnn_out... " - find "$HOME/hnn_out" -type d -exec chmod o+rwx {} \; >> "$LOGFILE" 2>&1 && \ - find "$HOME/hnn_out" -type f -exec chmod o+rw {} \; >> "$LOGFILE" 2>&1 - echo "done" | tee -a "$LOGFILE" - else - echo "Please make ${HOME}/hnn_out accessible by the user running docker (try making world readable/writable)" | tee -a "$LOGFILE" - fi - else - echo "done" | tee -a "$LOGFILE" - echo -e "\nFailure seems to be an issue with docker container." | tee -a "$LOGFILE" - echo "Please open an issue on github with $LOGFILE" | tee -a "$LOGFILE" - echo "https://github.com/jonescompneurolab/hnn/issues" | tee -a "$LOGFILE" - fi - echo "*failed*" | tee -a "$LOGFILE" - cleanup 2 + false else - echo "done" | tee -a "$LOGFILE" + echo "done" fi +} - if [[ $TRAVIS_TESTING -ne 0 ]] || [[ "$OS" == "linux" ]]; then - # This command will not work when a qemu VM is used with docker-machine. - # That is, only when TRAVIS_TESTING=1 and (OS="mac" or OS="windows") - COMMAND=(2 "touch" "$HOME/hnn_out/THIS_DIRECTORY_IS_SHARED_BETWEEN_DOCKER_AND_YOUR_OS") +function fix_hnn_out_perms_container_print { + # no arguments + # runs command in container to change permissions of hnn_out + check_var docker_cmd + check_var HNN_CONTAINER_NAME + check_var CONTAINER_TYPE + check_var LOGFILE + check_var SYSTEM_USER_DIR + + local __container_user + + if [[ "$CONTAINER_TYPE" == "windows" ]]; then + return 0 + else + __container_user="root" + fi + + check_hnn_out_perms_container_print + if [[ $? -ne 0 ]]; then + print_header_message "Updating hnn_out permissions inside container... " + COMMAND=(8 "$docker_cmd" "exec" "-u" "$__container_user" "$HNN_CONTAINER_NAME" "bash" "-c" \ + "find \"${SYSTEM_USER_DIR}/hnn_out\" -type d -exec chmod 777 {} \; && \ + find \"${SYSTEM_USER_DIR}/hnn_out\" -type f -exec chmod 666 {} \;") convert_COMMAND_to_escaped_array - output_run_command_arguments "${ESCAPED_COMMAND[@]}" &> /dev/null + MSYS_NO_PATHCONV=1 output_run_command_arguments "${ESCAPED_COMMAND[@]}" &> /dev/null + fail_on_bad_exit $? + + RETRY=1 + check_hnn_out_perms_container_print fi } @@ -763,6 +892,14 @@ function check_x_authenticated { # no arguments check_var DISPLAY check_var OS + check_var LOGFILE + + if [[ "$OS" == "wsl" ]]; then + find_program_print xset + if [[ $? -ne 0 ]]; then + echo -e "Skipping X server test: install xset with 'sudo apt-get install x11-xserver-utils'\n" | tee -a "$LOGFILE" + fi + fi if [[ ! "$OS" == "windows" ]]; then print_header_message "Checking if we can authenticate with X server... " @@ -809,7 +946,7 @@ function check_x_port_netcat { __send_args[$__index]=$(echo $__arg|sed "s/ /$ESC_STR/g") (( __index++ )) done - output_run_piped_command "${__send_args[@]}" > /dev/null + output_run_piped_command "${__send_args[@]}" >> "$LOGFILE" false fi } @@ -826,7 +963,7 @@ function check_x_port_container_fail { # set DISPLAY for OS __display=$(get_display_for_gui) if [[ $? -ne 0 ]]; then - cleanup $? + cleanup 2 fi print_header_message "Checking if X server is reachable from container... " @@ -959,7 +1096,10 @@ function copy_xauthority_file_fail { check_var XAUTHORITY check_var HNN_CONTAINER_NAME check_var LOGFILE + check_var CONTAINER_TYPE + local __container_xauthority_path + local __xauthority_path local __command_status local __user __user="$1" @@ -969,24 +1109,38 @@ function copy_xauthority_file_fail { cleanup 2 fi - print_header_message_short "Copying Xauthority file into container... " - if [[ $TRAVIS_TESTING -eq 1 ]] && ([[ "$OS" =~ "windows" ]] || [[ "$OS" == "wsl" ]]); then + __container_xauthority_path="/tmp/.Xauthority" + __xauthority_path="$XAUTHORITY" + + # exceptions for windows containers + if [[ "$CONTAINER_TYPE" == "windows" ]]; then + __container_xauthority_path="/home/hnn_user/.Xauthority" + + if [[ ! "$OS" == "wsl" ]]; then + COMMAND=(3 "cygpath.exe" "-w" "$__xauthority_path") + convert_COMMAND_to_escaped_array + __xauthority_path=$(output_run_command_arguments "${ESCAPED_COMMAND[@]}") + if [[ $? -ne 0 ]]; then + cleanup 2 + fi + fi + stop_container_silent - COMMAND=(4 "$docker_cmd" "cp" "$XAUTHORITY" "$HNN_CONTAINER_NAME:/windows/temp/.Xauthority") - else - COMMAND=(4 "$docker_cmd" "cp" "$XAUTHORITY" "$HNN_CONTAINER_NAME:/tmp/.Xauthority") fi + + print_header_message_short "Copying Xauthority file into container... " + COMMAND=(4 "$docker_cmd" "cp" "$__xauthority_path" "$HNN_CONTAINER_NAME:$__container_xauthority_path") convert_COMMAND_to_escaped_array - output_run_command_arguments "${ESCAPED_COMMAND[@]}" &> /dev/null + MSYS_NO_PATHCONV=1 output_run_command_arguments "${ESCAPED_COMMAND[@]}" &> /dev/null fail_on_bad_exit $? - if [[ $TRAVIS_TESTING -eq 1 ]] && ([[ "$OS" =~ "windows" ]] || [[ "$OS" == "wsl" ]]); then + if [[ "$CONTAINER_TYPE" == "windows" ]]; then start_container_silent return fi print_header_message "Changing Xauthority permissions in container... " COMMAND=(8 "$docker_cmd" "exec" "-u" "root" "$HNN_CONTAINER_NAME" "bash" "-c" \ - "chown $__user:hnn_group /tmp/.Xauthority && chmod g+rw /tmp/.Xauthority") + "chown $__user:hnn_group \"$__container_xauthority_path\" && chmod g+rw \"$__container_xauthority_path\"") convert_COMMAND_to_escaped_array output_run_command_arguments "${ESCAPED_COMMAND[@]}" &> /dev/null fail_on_bad_exit $? @@ -994,89 +1148,208 @@ function copy_xauthority_file_fail { } function copy_hnn_source_fail { - # copies $PWD to hnn_source_code and sets permissions for hnn_out directory - # for the appropriate user + # copies $PWD to hnn_source_code check_var docker_cmd - check_var XAUTHORITY check_var HNN_CONTAINER_NAME - check_var USE_SSH - check_var UID check_var LOGFILE + check_var CONTAINER_TYPE - local __user - - if [[ $USE_SSH -eq 0 ]]; then - __user="$UID" - else - __user="hnn_user" - fi + local __source_code_path + local __current_directory if [[ ! -e "$PWD/hnn.py" ]]; then echo -e "\nBad hnn_source_code directory at $PWD" | tee -a "$LOGFILE" cleanup 2 fi - print_header_message "Removing old hnn_source_code from container... " - if [[ $TRAVIS_TESTING -eq 1 ]] && ([[ "$OS" =~ "windows" ]] || [[ "$OS" == "wsl" ]]); then - COMMAND=(6 "$docker_cmd" "exec" "$HNN_CONTAINER_NAME" "bash" "-c" "rm -rf /home/hnn_user/hnn_source_code") + if [[ "$CONTAINER_TYPE" == "linux" ]]; then + __source_code_path="/home/hnn_user/hnn_source_code" else - COMMAND=(8 "$docker_cmd" "exec" "-u" "root" "$HNN_CONTAINER_NAME" "bash" "-c" "rm -rf /home/hnn_user/hnn_source_code") + __source_code_path="/c/home/hnn_user/hnn_source_code" fi + + print_header_message "Removing old hnn_source_code from container... " + COMMAND=(6 "$docker_cmd" "exec" "$HNN_CONTAINER_NAME" "bash" "-c" "rm -rf $__source_code_path/*") convert_COMMAND_to_escaped_array output_run_command_arguments "${ESCAPED_COMMAND[@]}" &> /dev/null fail_on_bad_exit $? - if [[ $TRAVIS_TESTING -eq 1 ]] && ([[ "$OS" =~ "windows" ]] || [[ "$OS" == "wsl" ]]); then + # exceptions for windows containers + if [[ "$CONTAINER_TYPE" == "windows" ]]; then + COMMAND=(3 "cygpath.exe" "-w" "$PWD") + convert_COMMAND_to_escaped_array + __current_directory=$(output_run_command_arguments "${ESCAPED_COMMAND[@]}") + if [[ $? -ne 0 ]]; then + cleanup 2 + fi + __current_directory="$__current_directory\." + stop_container_silent + else + __current_directory="$PWD/." fi print_header_message_short "Copying hnn_source_code into container... " - COMMAND=(4 "$docker_cmd" "cp" "$PWD" "$HNN_CONTAINER_NAME:/home/hnn_user/hnn_source_code") + COMMAND=(4 "$docker_cmd" "cp" "$__current_directory" "$HNN_CONTAINER_NAME:/home/hnn_user/hnn_source_code/") convert_COMMAND_to_escaped_array - output_run_command_arguments "${ESCAPED_COMMAND[@]}" &> /dev/null + MSYS_NO_PATHCONV=1 output_run_command_arguments "${ESCAPED_COMMAND[@]}" &> /dev/null fail_on_bad_exit $? - if [[ $TRAVIS_TESTING -eq 1 ]] && ([[ "$OS" =~ "windows" ]] || [[ "$OS" == "wsl" ]]); then + if [[ "$CONTAINER_TYPE" == "windows" ]]; then start_container_silent fi +} - if [[ $TRAVIS_TESTING -eq 0 ]] || ([[ ! "$OS" =~ "windows" ]] && [[ ! "$OS" == "wsl" ]]); then - print_header_message "Changing hnn directory permissions in container... " +function copy_hnn_out_fail { + # copies $SYSTEM_USER_DIR/hnn_out to hnn_source_code + check_var docker_cmd + check_var HNN_CONTAINER_NAME + check_var LOGFILE + check_var CONTAINER_TYPE + check_var TRAVIS_TESTING + + local __hnn_out + __hnn_out="$SYSTEM_USER_DIR/hnn_out" + + if [[ ! -e "$__hnn_out" ]]; then + echo -e "\nError: hnn_out doesn't exist at $__hnn_out" | tee -a "$LOGFILE" + cleanup 2 + fi + + if [[ "$OS" == "mac" ]] && [[ $TRAVIS_TESTING -eq 1 ]]; then + print_header_message "Creating hnn_out in container... " COMMAND=(8 "$docker_cmd" "exec" "-u" "root" "$HNN_CONTAINER_NAME" "bash" "-c" \ - "chown -R $__user /home/hnn_user/hnn_source_code && chown $__user $SYSTEM_USER_DIR/hnn_out") + "mkdir -p \"$__hnn_out/data\" && \ + mkdir -p \"$__hnn_out/param\" && \ + chown -R hnn_user:hnn_group \"$__hnn_out\" && \ + chmod 777 -R \"$__hnn_out\"") convert_COMMAND_to_escaped_array output_run_command_arguments "${ESCAPED_COMMAND[@]}" &> /dev/null - fail_on_bad_exit $? + fail_on_bad_exit $? + else + print_header_message "Removing old hnn_out from container... " + + if [[ "$CONTAINER_TYPE" == "windows" ]]; then + COMMAND=(6 "$docker_cmd" "exec" "$HNN_CONTAINER_NAME" "bash" "-c" "rm -rf $__hnn_out/*") + else + COMMAND=(8 "$docker_cmd" "exec" "-u" "root" "$HNN_CONTAINER_NAME" "bash" "-c" "rm -rf $__hnn_out/*") + fi + convert_COMMAND_to_escaped_array + output_run_command_arguments "${ESCAPED_COMMAND[@]}" &> /dev/null + fail_on_bad_exit $? + fi + + __hnn_out_contents="$__hnn_out/." + # exceptions for windows containers + if [[ "$CONTAINER_TYPE" == "windows" ]]; then + COMMAND=(3 "cygpath.exe" "-w" "$__hnn_out_contents") + convert_COMMAND_to_escaped_array + __hnn_out_contents=$(output_run_command_arguments "${ESCAPED_COMMAND[@]}") + if [[ $? -ne 0 ]]; then + cleanup 2 + fi + + # remove leading /c/ + __hnn_out="$(echo $__hnn_out | sed 's/^\/[cC]\(.*\)$/\1/')" + stop_container_silent fi + print_header_message_short "Copying hnn_out into container... " + COMMAND=(4 "$docker_cmd" "cp" "$__hnn_out_contents" "$HNN_CONTAINER_NAME:$__hnn_out") + convert_COMMAND_to_escaped_array + MSYS_NO_PATHCONV=1 output_run_command_arguments "${ESCAPED_COMMAND[@]}" &> /dev/null + fail_on_bad_exit $? + if [[ "$CONTAINER_TYPE" == "windows" ]]; then + start_container_silent + fi +} + +function change_hnn_source_perms_fail { + check_var docker_cmd + check_var HNN_CONTAINER_NAME + check_var CONTAINER_TYPE + check_var USE_SSH + check_var UID + + local __container_user + local __user + + if [[ $USE_SSH -eq 0 ]]; then + __user="$UID" + else + __user="hnn_user" + fi + + if [[ "$CONTAINER_TYPE" == "windows" ]]; then + return 0 + else + __container_user="root" + __source_code_path="/home/hnn_user/hnn_source_code" + fi + print_header_message "Changing hnn_source_code permissions in container... " + COMMAND=(8 "$docker_cmd" "exec" "-u" "$__container_user" "$HNN_CONTAINER_NAME" "bash" "-c" \ + "chown -R $__user $__source_code_path") + convert_COMMAND_to_escaped_array + output_run_command_arguments "${ESCAPED_COMMAND[@]}" &> /dev/null + fail_on_bad_exit $? } -function copy_ssh_files_to_running_container_fail { +function copy_ssh_files_to_container_fail { # no arguments + # *** This probably won't work with windows containers because paths are in UNIX format ***** check_var docker_cmd check_var SSH_AUTHKEYS check_var SSH_PUBKEY + check_var CONTAINER_TYPE + + local __authkeys + local __pubkey + __authkeys="$SSH_AUTHKEYS" + __pubkey="$SSH_PUBKEY" + + if [[ "$CONTAINER_TYPE" == "windows" ]]; then + COMMAND=(3 "cygpath.exe" "-w" "$__authkeys") + convert_COMMAND_to_escaped_array + __authkeys=$(output_run_command_arguments "${ESCAPED_COMMAND[@]}") + if [[ $? -ne 0 ]]; then + cleanup 2 + fi + + COMMAND=(3 "cygpath.exe" "-w" "$__pubkey") + convert_COMMAND_to_escaped_array + __pubkey=$(output_run_command_arguments "${ESCAPED_COMMAND[@]}") + if [[ $? -ne 0 ]]; then + cleanup 2 + fi - if [[ $TRAVIS_TESTING -eq 1 ]] && ([[ "$OS" =~ "windows" ]] || [[ "$OS" == "wsl" ]]); then stop_container_silent fi + print_header_message_short "Copying authorized_keys file into container... " - COMMAND=(4 "$docker_cmd" "cp" "$SSH_AUTHKEYS" "$HNN_CONTAINER_NAME:/home/hnn_user/.ssh/authorized_keys") + COMMAND=(4 "$docker_cmd" "cp" "$__authkeys" "$HNN_CONTAINER_NAME:/home/hnn_user/.ssh/authorized_keys") convert_COMMAND_to_escaped_array - output_run_command_arguments "${ESCAPED_COMMAND[@]}" &> /dev/null + MSYS_NO_PATHCONV=1 output_run_command_arguments "${ESCAPED_COMMAND[@]}" &> /dev/null fail_on_bad_exit $? print_header_message_short "Copying known_hosts file into container... " - COMMAND=(4 "$docker_cmd" "cp" "$SSH_PUBKEY" "$HNN_CONTAINER_NAME:/home/hnn_user/.ssh/known_hosts") + COMMAND=(4 "$docker_cmd" "cp" "$__pubkey" "$HNN_CONTAINER_NAME:/home/hnn_user/.ssh/known_hosts") convert_COMMAND_to_escaped_array - output_run_command_arguments "${ESCAPED_COMMAND[@]}" &> /dev/null + MSYS_NO_PATHCONV=1 output_run_command_arguments "${ESCAPED_COMMAND[@]}" &> /dev/null fail_on_bad_exit $? - if [[ $TRAVIS_TESTING -eq 1 ]] && ([[ "$OS" =~ "windows" ]] || [[ "$OS" == "wsl" ]]); then + if [[ "$CONTAINER_TYPE" == "windows" ]]; then start_container_silent + # don't bother updating unix permissions + return 0 fi - print_header_message_short "Updating permissions on ssh files in container... " + print_header_message_short "Updating ownership on ssh files in container... " COMMAND=(8 "$docker_cmd" "exec" "-u" "root" "$HNN_CONTAINER_NAME" "bash" "-c" "chown -R hnn_user /home/hnn_user/.ssh") convert_COMMAND_to_escaped_array output_run_command_arguments "${ESCAPED_COMMAND[@]}" &> /dev/null fail_on_bad_exit $? + + print_header_message_short "Updating permissions on ssh authorized_keys file in container... " + COMMAND=(8 "$docker_cmd" "exec" "-u" "root" "$HNN_CONTAINER_NAME" "bash" "-c" "chmod 600 /home/hnn_user/.ssh/authorized_keys") + convert_COMMAND_to_escaped_array + output_run_command_arguments "${ESCAPED_COMMAND[@]}" &> /dev/null + fail_on_bad_exit $? } function kill_xquartz { @@ -1144,6 +1417,7 @@ function get_xquartz_port { let __timeout=30 while [[ $__timeout -gt 0 ]]; do + log_header_message "Getting running Xquartz processes... " __command="pgrep Xquartz" __pid=$(output_run_command "$__command") if [[ $? -eq 0 ]]; then @@ -1155,10 +1429,12 @@ function get_xquartz_port { __xquartz_display=$(ps $__pid|grep $__pid|sed 's/.*\(\:[0-9]\{1,\}\).*/\1/') __display_int=$(echo $__xquartz_display|sed 's/\:\([0-9]\{1,\}\)/\1/') if [[ -e "/tmp/.X11-unix/X${__display_int}" ]]; then + echo "done" >> "$LOGFILE" echo "Started XQuartz on DISPLAY $__xquartz_display" >> "$LOGFILE" break fi fi + echo "done" >> "$LOGFILE" sleep 1 (( __timeout-- )) done @@ -1189,7 +1465,7 @@ function start_xquartz { re='^[0-9]+$' __port=$(get_xquartz_port) if [[ $? -ne 0 ]]; then - exit $? + exit 2 elif ! [[ $__port =~ $re ]] ; then echo "bad xquartz port number \"$__port\"" >> "$LOGFILE" false @@ -1247,13 +1523,14 @@ function docker_pull { check_var OS check_var HNN_DOCKER_IMAGE check_var TRAVIS_TESTING + check_var CONTAINER_TYPE local __command local __docker_container local __last_used_image local __docker_image - if [[ "$OS" == "windows" ]] || [[ "$OS" == "wsl" ]]; then + if [[ "$CONTAINER_TYPE" == "windows" ]]; then __docker_image="${HNN_DOCKER_IMAGE}:win64" else __docker_image="${HNN_DOCKER_IMAGE}" @@ -1314,15 +1591,15 @@ function retry_docker_pull { fi } -function docker_run_print { +function docker_run_fail { # no arguments # will run the appropriate docker command # to leave a running containers check_var TRAVIS_TESTING check_var LOGFILE - check_var RETRY check_var OS check_var USE_SSH + check_var CONTAINER_TYPE local __timeout local __started @@ -1332,24 +1609,15 @@ function docker_run_print { local __arg local __send_args local __docker_image + local __home + local __hnn_out + local __container_xauthority __timeout=20 - if [[ $TRAVIS_TESTING -eq 1 ]]; then - find_existing_container_print - if [[ $? -eq "0" ]]; then - print_header_message "Removing old container... " - remove_container_fail - fi - elif [[ $RETRY -eq 1 ]]; then - echo "Removing old container might resolve failure. Verification required." | tee -a "$LOGFILE" - prompt_remove_container_fail - echo -n "(retry) " | tee -a "$LOGFILE" - RETRY=0 - fi - print_header_message "Starting HNN container... " + print_header_message "Creating HNN container... " - if [[ "$OS" =~ "windows" ]] || [[ "$OS" == "wsl" ]]; then + if [[ "$CONTAINER_TYPE" == "windows" ]]; then # Interesting that HOME still gets turned into a windows path. Easy to avoid # by passing the HOME variable under a new name HOME_UNIX export HOME_UNIX="${HOME}" @@ -1358,67 +1626,52 @@ function docker_run_print { __docker_image="${HNN_DOCKER_IMAGE}" fi - if [[ $TRAVIS_TESTING -eq 1 ]] && [[ "$OS" =~ "linux" ]]; then - # Importantly, the volume containing the current directory is bind mounted into - # the container and seen as /home/hnn_user/hnn_source_code, which enables tests - # inside the container to use code which has not yet been merged and been built - # as part of a container on Docker Hub. - COMMAND=(20 "${docker_cmd}" "run" "-d" \ - "-p" "22" \ - "-v" "/tmp/.X11-unix:/tmp/.X11-unix" \ - "-v" "$(pwd):/home/hnn_user/hnn_source_code" \ - "-v" "$HOME/hnn_out:$HOME/hnn_out" \ - "--env" "XAUTHORITY=/tmp/.Xauthority" "--env" "SYSTEM_USER_DIR=$HOME" \ - "--name" "$HNN_CONTAINER_NAME" "$__docker_image" "sleep" "infinity") - elif [[ $TRAVIS_TESTING -eq 1 ]] && [[ "$OS" =~ "windows" ]]; then - HNN_OUT=$(cygpath -w "$HOME/hnn_out") - COMMAND=(16 "${docker_cmd}" "run" "-d" \ - "-p" "22" \ - "-v" "$HNN_OUT:$HNN_OUT" \ - "--env" "XAUTHORITY=/c/windows/temp/.Xauthority" "--env" "SYSTEM_USER_DIR=$HOME" \ - "--name" "$HNN_CONTAINER_NAME" "$__docker_image" "sleep" "infinity") - elif [[ $TRAVIS_TESTING -eq 1 ]] && [[ "$OS" == "wsl" ]]; then - # we can't access the wsl directory, so don't create a bind-mounted volume - COMMAND=(14 "${docker_cmd}" "run" "-d" \ - "-p" "22" \ - "--env" "XAUTHORITY=/c/windows/temp/.Xauthority" "--env" "SYSTEM_USER_DIR=$HOME" \ - "--name" "$HNN_CONTAINER_NAME" "$__docker_image" "sleep" "infinity") - elif [[ $TRAVIS_TESTING -eq 1 ]]; then - # This binds port 22 in the container (for sshd) to 5000 on the host, which on docker toolbox - # is a VM. We can then configure the VM with docker-machine to allow connections to port 5000 from - # the base OS. Also connect port 6000 to allow for xforwarding of the xserver. - COMMAND=(18 "${docker_cmd}" "run" "-d" \ - "-p" "6000:6000" "-p" "5000:22" \ - "-v" "$HOME/hnn_out:$HOME/hnn_out" \ - "--env" "XAUTHORITY=/tmp/.Xauthority" "--env" "SYSTEM_USER_DIR=$HOME" \ - "--name" "$HNN_CONTAINER_NAME" "$__docker_image" "sleep" "infinity") - elif [[ "$OS" =~ "linux" ]]; then - COMMAND=(18 "${docker_cmd}" "run" "-d" \ + # exceptions for windows containers + __hnn_out="$SYSTEM_USER_DIR/hnn_out" + if [[ "$CONTAINER_TYPE" == "windows" ]]; then + COMMAND=(3 "cygpath.exe" "-w" "$__hnn_out") + convert_COMMAND_to_escaped_array + __hnn_out=$(output_run_command_arguments "${ESCAPED_COMMAND[@]}") + if [[ $? -ne 0 ]]; then + cleanup 2 + fi + fi + + __container_xauthority="/tmp/.Xauthority" + if [[ "$CONTAINER_TYPE" == "windows" ]]; then + __container_xauthority="/c/home/hnn_user/.Xauthority" + fi + + if [[ "$OS" == "mac" ]] && [[ $TRAVIS_TESTING -eq 1 ]]; then + # Workarounds for qemu docker-machine driver that doesn't support port forwarding and volume + # mounts properly. + # This binds port 22 in the container (for sshd) to 5000 on the host, which on docker toolbox + # is a VM. We can then configure the VM with docker-machine to allow connections to port 5000 from + # the base OS. Also connect port 6000 to allow for xforwarding of the xserver. + COMMAND=(14 "${docker_cmd}" "run" "-d" \ + "-p" "6000:6000" "-p" "5000:22" \ + "--env" "XAUTHORITY=$__container_xauthority" "--env" "SYSTEM_USER_DIR=$SYSTEM_USER_DIR" \ + "--name" "$HNN_CONTAINER_NAME" "$__docker_image") + elif [[ "$OS" == "linux" ]]; then + COMMAND=(16 "${docker_cmd}" "run" "-d" \ "-p" "22" \ "-v" "/tmp/.X11-unix:/tmp/.X11-unix" \ - "-v" "$HOME/hnn_out:$HOME/hnn_out" \ - "--env" "XAUTHORITY=/tmp/.Xauthority" "--env" "SYSTEM_USER_DIR=$HOME" \ - "--name" "$HNN_CONTAINER_NAME" "$__docker_image" "sleep" "infinity") - elif [[ "$OS" == "wsl" ]]; then - # we can't access the wsl directory, so don't create a bind-mounted volume - COMMAND=(14 "${docker_cmd}" "run" "-d" \ - "-p" "22" \ - "--env" "XAUTHORITY=/tmp/.Xauthority" "--env" "SYSTEM_USER_DIR=$HOME" \ - "--name" "$HNN_CONTAINER_NAME" "$__docker_image" "sleep" "infinity") - else - COMMAND=(16 "${docker_cmd}" "run" "-d" \ - "-p" "22" \ - "-v" "$HOME/hnn_out:$HOME/hnn_out" \ - "--env" "XAUTHORITY=/tmp/.Xauthority" "--env" "SYSTEM_USER_DIR=$HOME" \ - "--name" "$HNN_CONTAINER_NAME" "$__docker_image" "sleep" "infinity") + "-v" "$__hnn_out:$__hnn_out" \ + "--env" "XAUTHORITY=$__container_xauthority" "--env" "SYSTEM_USER_DIR=$SYSTEM_USER_DIR" \ + "--name" "$HNN_CONTAINER_NAME" "$__docker_image") + else # windows and mac (TRAVIS_TESTING=0) + COMMAND=(14 "${docker_cmd}" "run" "-d" \ + "-p" "22" \ + "-v" "$__hnn_out:$__hnn_out" \ + "--env" "XAUTHORITY=$__container_xauthority" "--env" "SYSTEM_USER_DIR=$SYSTEM_USER_DIR" \ + "--name" "$HNN_CONTAINER_NAME" "$__docker_image") fi convert_COMMAND_to_escaped_array MSYS_NO_PATHCONV=1 output_run_command_arguments "${ESCAPED_COMMAND[@]}" &> /dev/null if [[ $? -ne 0 ]]; then echo "*failed*" | tee -a "$LOGFILE" - false - return + cleanup 2 fi # make sure "docker ps | grep HNN_CONTAINER_NAME" succeeeds @@ -1426,16 +1679,10 @@ function docker_run_print { if [[ ! "$__started" =~ "1" ]]; then echo "*failed*" | tee -a "$LOGFILE" echo "Waited for $__timeout seconds for container to start" >> "$LOGFILE" - false - return + cleanup 2 else echo "done" | tee -a "$LOGFILE" fi - - # copy ssh auth files for TRAVIS_TESTING=1 - if [[ $USE_SSH -eq 1 ]]; then - copy_ssh_files_to_running_container_fail - fi } function print_sshd_log { @@ -1470,38 +1717,36 @@ function generate_xauth_keys_fail { fail_on_bad_exit $? } -function get_xauth_keys { +function get_host_xauth_keys { check_var LOGFILE check_var xauth_cmd check_var XAUTHORITY check_var DISPLAY + check_var DEBUG local __output + local __suppress_output + local __keys local __command_status - # don't use output_run_command_arguments because we don't want to log keys - # to log file - echo -e "\n ** Command: \"${xauth_cmd}\" -f \"$XAUTHORITY\" nlist $DISPLAY" >> "$LOGFILE" - echo -n " ** Stderr: " >> "$LOGFILE" - - if [[ "$OS" =~ "windows" ]]; then - __output=$("${xauth_cmd}" -f "$XAUTHORITY" nlist $DISPLAY 2>> "$LOGFILE") + if [[ $DEBUG -eq 1 ]]; then + __suppress_output=0 else - __output=$("${xauth_cmd}" -f "$XAUTHORITY" nlist $DISPLAY 2>> "$LOGFILE") + # don't display the keys in the log file + __suppress_output=1 fi - __command_status=$? - # don't print keys in log - if [ -n "$__output" ]; then - echo " ** Stdout: ** suppresed **" >> "$LOGFILE" + log_header_message "Retrieving host xauth keys... " + COMMAND=(6 "${xauth_cmd}" "-f" "$XAUTHORITY" "-ni" "nlist" "$DISPLAY") + convert_COMMAND_to_escaped_array + __keys=$(SUPPRESS_OUTPUT=$__suppress_output output_run_command_arguments "${ESCAPED_COMMAND[@]}") + __command_status=$? + if [[ $__command_status -ne 0 ]]; then + echo "*failed*" >> "$LOGFILE" + exit $__command_status else - echo " ** Stdout: " >> "$LOGFILE" - fi - echo "$__output" - if [[ $__command_status -eq 0 ]]; then - true - else - false + echo "done" >> "$LOGFILE" + echo "$__keys" fi } @@ -1513,13 +1758,13 @@ function check_xauth_keys_print { local __command_status print_header_message "Checking for X11 authentication keys... " - __output=$(get_xauth_keys) + __output=$(get_host_xauth_keys) __command_status=$? + echo >> "$LOGFILE" if [[ $__command_status -ne 0 ]]; then echo "*failed*" | tee -a "$LOGFILE" exit $__command_status - fi - if [[ -z "$__output" ]]; then + elif [[ -z "$__output" ]]; then echo "no valid keys" | tee -a "$LOGFILE" false else @@ -1590,6 +1835,8 @@ function get_display_for_gui { # no arguments # set DISPLAY environment variable for the appropriate OS and/or testing environment check_var DOCKER_TOOLBOX + check_var CONTAINER_TYPE + check_var TRAVIS_TESTING local __display local __port @@ -1601,23 +1848,37 @@ function get_display_for_gui { fi # set DISPLAY for GUI - if [[ "$OS" =~ "linux" ]]; then - # linux can use direct port - if [[ $TRAVIS_TESTING -eq 1 ]]; then - [[ $DISPLAY ]] && __display=$DISPLAY || __display=":$__port" + if [[ "$CONTAINER_TYPE" == "windows" ]]; then + log_header_message "Getting IP for DISPLAY from container... " + COMMAND=(6 "$docker_cmd" "exec" "$HNN_CONTAINER_NAME" "bash" "-c" "ipconfig.exe | grep 'Default Gateway'| sed -e 's/^.*:[ \t]*\(.*\)/\1/'") + convert_COMMAND_to_escaped_array + __ip=$(output_run_command_arguments "${ESCAPED_COMMAND[@]}") + if [[ $? -ne 0 ]]; then + echo "*failed*" >> "$LOGFILE" + exit 2 else - __display=":$__port" + echo "done" >> "$LOGFILE" fi + __display="$__ip:$__port" else - if [[ $TRAVIS_TESTING -eq 1 ]]; then - # qemu driver - __display="10.0.2.2:$__port" - elif [[ $DOCKER_TOOLBOX -eq 1 ]]; then - # virtualbox driver - __display="192.168.99.1:$__port" + if [[ "$OS" =~ "linux" ]]; then + # linux can use direct port + if [[ $TRAVIS_TESTING -eq 1 ]]; then + [[ $DISPLAY ]] && __display=$DISPLAY || __display=":$__port" + else + __display=":$__port" + fi else - # docker desktop - __display="host.docker.internal:$__port" + if [[ $TRAVIS_TESTING -eq 1 ]]; then + # qemu driver + __display="10.0.2.2:$__port" + elif [[ $DOCKER_TOOLBOX -eq 1 ]]; then + # virtualbox driver + __display="192.168.99.1:$__port" + else + # docker desktop + __display="host.docker.internal:$__port" + fi fi fi @@ -1625,21 +1886,37 @@ function get_display_for_gui { } -function get_host_xauth_key { +function get_valid_host_xauth_key { # no arguments # retrieves xauth key from host - check_var LOGFILE check_var DISPLAY check_var xauth_cmd + check_var XAUTHORITY + check_var DEBUG + local __keys local __key + local __command_status - echo -e "\n ** Command: xauth -ni nlist $DISPLAY | grep '^fff' | head -1 | awk '{print \$NF}'" >> "$LOGFILE" - __key=$("${xauth_cmd}" -ni nlist $DISPLAY | grep '^fff' | head -1 | awk '{print $NF}' 2>> "$LOGFILE") + __keys=$(get_host_xauth_keys) + __command_status=$? if [[ $? -ne 0 ]]; then - return 1 + return $__command_status else - echo $__key + log_header_message "Validating host xauth keys... " + __key="$(echo $__keys | grep 'ffff' | head -1 | awk '{print $NF}')" + if [[ -z "$__key" ]]; then + # no output means no valid keys + echo "*failed*" >> "$LOGFILE" + echo -e " ** Error: no valid key **" >> "$LOGFILE" + return 1 + else + echo "done" >> "$LOGFILE" + if [[ $DEBUG -eq 1 ]]; then + echo "Valid key: =START=$__key=END=" >> "$LOGFILE" + fi + echo "$__key" + fi fi } @@ -1649,26 +1926,54 @@ function get_container_xauth_key { check_args "$@" $# 1 - check_var LOGFILE check_var docker_cmd check_var HNN_CONTAINER_NAME + check_var DEBUG + local __keys local __key local __user local __display + local __suppress_output + local __command_status __user="$1" __display=$(get_display_for_gui) if [[ $? -ne 0 ]]; then - cleanup $? + cleanup 2 fi - echo -e "\n ** Command: \"$docker_cmd\" exec -u \"$__user\" \"$HNN_CONTAINER_NAME\" bash -c \"timeout 5 xauth -ni nlist \"$__display\" | grep '^ffff' | head -1 | awk '{print \$NF}'\"" >> "$LOGFILE" - echo -n " ** Stderr: " >> "$LOGFILE" - __key=$("$docker_cmd" exec -u "$__user" "$HNN_CONTAINER_NAME" bash -c "timeout 5 xauth -ni nlist "$__display" | grep '^ffff' | head -1 | awk '{print \$NF}'" 2>> "$LOGFILE") - if [[ $? -ne 0 ]]; then - return $? + log_header_message "Retrieving container xauth keys... " + COMMAND=(8 "$docker_cmd" exec "-u" "$__user" "$HNN_CONTAINER_NAME" "bash" "-c" "timeout 5 xauth -ni nlist $__display") + convert_COMMAND_to_escaped_array + # don't display the keys in the log file + if [[ $DEBUG -eq 1 ]]; then + __suppress_output=0 else - echo $__key + __suppress_output=1 + fi + __keys=$(SUPPRESS_OUTPUT=$__suppress_output output_run_command_arguments "${ESCAPED_COMMAND[@]}") + __command_status=$? + if [[ $__command_status -ne 0 ]]; then + echo "*failed*" >> "$LOGFILE" + exit $__command_status + else + echo "done" >> "$LOGFILE" + + log_header_message "Validating container xauth keys... " + __key="$(echo $__keys | grep 'ffff' | head -1 | awk '{print $NF}')" + if [[ -z "$__key" ]]; then + # no output means no valid keys + echo "*failed*" >> "$LOGFILE" + echo -e " ** Error: no valid key **" >> "$LOGFILE" + return 1 + else + echo "done" >> "$LOGFILE" + fi + + if [[ $DEBUG -eq 1 ]]; then + echo "Valid key: =START=$__key=END=" >> "$LOGFILE" + fi + echo "$__key" fi } @@ -1678,44 +1983,42 @@ function check_container_xauth_print { check_var LOGFILE check_var RETRY + check_var CONTAINER_TYPE local __user + local __command_status local __host_key - __user=$1 - __host_key=$(get_host_xauth_key) - if [[ $? -ne 0 ]]; then - exit $? - elif [[ -z $__host_key ]]; then - echo $__host_key - # no output means no valid keys - echo -e " ** Error: no valid key **" >> "$LOGFILE" - return 1 - fi + local __key - if [[ $TRAVIS_TESTING -eq 1 ]] && ([[ "$OS" =~ "windows" ]] || [[ "$OS" == "wsl" ]]); then - # always return success because we can't run xauth within container - return 0 + __user=$1 + __host_key=$(get_valid_host_xauth_key) + __command_status=$? + if [[ $__command_status -ne 0 ]]; then + cleanup 2 fi if [[ $RETRY -eq 1 ]]; then echo -n "(retry) " | tee -a "$LOGFILE" RETRY=0 fi - print_header_message "Checking that xauth key in container matches... " - __key=$(get_container_xauth_key $1) + + if [[ "$CONTAINER_TYPE" == "windows" ]]; then + __key=$(get_container_xauth_key "hnn_user") + else + __key=$(get_container_xauth_key $1) + fi __command_status=$? + + + print_header_message "Checking that xauth key in container matches... " if [[ $__command_status -ne 0 ]]; then echo "*failed*" | tee -a "$LOGFILE" echo -e " ** Error: could not retrieve keys from container **" >> "$LOGFILE" - exit $__command_status - elif [[ -z $__key ]]; then - # no output means no valid keys - echo "*failed*" | tee -a "$LOGFILE" - echo -e " ** Error: no valid key **" >> "$LOGFILE" return 1 elif [[ "$__host_key" == "$__key" ]]; then echo "done" | tee -a "$LOGFILE" else + echo "*$__host_key* != *$__key*" >> "$LOGFILE" echo "*failed*" | tee -a "$LOGFILE" echo -e " ** Error: key mismatch **" >> "$LOGFILE" return 1 @@ -1758,7 +2061,6 @@ function setup_xauthority_in_container_fail { # make sure that xauth works in container now check_container_xauth_print "$__user" if [[ $? -ne 0 ]]; then - echo "*failed*" | tee -a "$LOGFILE" cleanup 2 fi fi @@ -1799,13 +2101,13 @@ function check_xquartz_listening { [[ $OS ]] || export OS=$(get_os) if [[ $? -ne 0 ]]; then - cleanup $? + cleanup 2 fi # before assuming port 0, check if xquartz is already running __current_port=$(get_xquartz_port) if [[ $? -ne 0 ]]; then - exit $? + exit 2 elif [[ -z $__current_port ]]; then __current_port="0" fi @@ -1819,7 +2121,7 @@ function check_xquartz_listening { fi __current_port=$(restart_xquartz_fail) if [[ $? -ne 0 ]]; then - exit $? + exit 2 fi sleep 1 (( __retry++ )) @@ -1927,18 +2229,18 @@ function generate_ssh_auth_keys_fail { cleanup 2 fi - echo -e "\n ** Command: { echo -n 'command=\"/home/hnn_user/start_hnn.sh\" '; cat \"$SSH_PUBKEY\"; } > \"$SSH_AUTHKEYS\"" >> "$LOGFILE" - echo -n " ** Stderr: " >> "$LOGFILE" - { echo -n 'command="/home/hnn_user/start_hnn.sh" '; cat "$SSH_PUBKEY"; } > "$SSH_AUTHKEYS" 2>> "$LOGFILE" + COMMAND=(3 "chmod" "600" "$SSH_PRIVKEY") + convert_COMMAND_to_escaped_array + output_run_command_arguments "${ESCAPED_COMMAND[@]}" &> /dev/null if [[ $? -ne "0" ]]; then echo "*failed*" | tee -a "$LOGFILE" cleanup 2 fi echo >> "$LOGFILE" - COMMAND=(3 "chmod" "600" "$SSH_AUTHKEYS") - convert_COMMAND_to_escaped_array - output_run_command_arguments "${ESCAPED_COMMAND[@]}" &> /dev/null + echo -e "\n ** Command: { echo -n 'command=\"/home/hnn_user/start_hnn.sh\" '; cat \"$SSH_PUBKEY\"; } > \"$SSH_AUTHKEYS\"" >> "$LOGFILE" + echo -n " ** Stderr: " >> "$LOGFILE" + { echo -n 'command="/home/hnn_user/start_hnn.sh" '; cat "$SSH_PUBKEY"; } > "$SSH_AUTHKEYS" 2>> "$LOGFILE" if [[ $? -ne "0" ]]; then echo "*failed*" | tee -a "$LOGFILE" cleanup 2 @@ -2007,10 +2309,13 @@ function check_docker_working_print { COMMAND=(4 "timeout" "5" "$docker_cmd" version) convert_COMMAND_to_escaped_array __output=$(output_run_command_arguments "${ESCAPED_COMMAND[@]}") - if [[ ! -z $__output ]] && [[ $? -ne "0" ]]; then + if [[ $? -ne 0 ]]; then echo "*failed*" | tee -a "$LOGFILE" false - elif [[ $? -eq "124" ]]; then + elif [[ -z "$__output" ]]; then + echo "*failed*" | tee -a "$LOGFILE" + false + elif [[ $? -eq 124 ]]; then echo "*failed*" | tee -a "$LOGFILE" echo "Error: timed out connecting to Docker. Please check Docker install" | tee -a "$LOGFILE" false @@ -2019,6 +2324,43 @@ function check_docker_working_print { fi } +function get_docker_container_type { + check_var LOGFILE + check_var docker_cmd + + local __output + + print_header_message "Checking Docker container type... " + COMMAND=(4 "timeout" "5" "$docker_cmd" info) + convert_COMMAND_to_escaped_array + __output=$(output_run_command_arguments "${ESCAPED_COMMAND[@]}") + if [[ $? -ne 0 ]]; then + echo "*failed*" | tee -a "$LOGFILE" + cleanup 2 + elif [[ -z "$__output" ]]; then + echo "*failed*" | tee -a "$LOGFILE" + cleanup 2 + elif [[ $? -eq 124 ]]; then + echo "*failed*" | tee -a "$LOGFILE" + echo "Error: timed out connecting to Docker. Please check Docker install" | tee -a "$LOGFILE" + cleanup 2 + else + if [[ "$(echo $__output | grep "Storage Driver:")" =~ "windowsfilter" ]]; then + CONTAINER_TYPE="windows" + else + CONTAINER_TYPE="linux" + fi + echo "$CONTAINER_TYPE" | tee -a "$LOGFILE" + fi + + if [[ "$CONTAINER_TYPE" == "windows" ]] && [[ "$OS" == "wsl" ]]; then + echo "** Windows containers not supported in WSL. Use Linux containers. **" | tee -a "$LOGFILE" + cleanup 1 + fi + + export CONTAINER_TYPE +} + function remove_hnn_image_fail { check_var docker_cmd check_var HNN_DOCKER_IMAGE @@ -2045,13 +2387,29 @@ function remove_hnn_image_fail { function check_vcxsrv_running_print { check_var LOGFILE + check_var TRAVIS_TESTING print_header_message_short "Checking if VcXsrv is running... " - export VCXSRV_PID=$(tasklist | grep vcxsrv | awk '{print $2}' 2> /dev/null) + if [[ "$OS" == "wsl" ]]; then + TASKS=$(powershell.exe -c '& tasklist') + if [[ $? -ne 0 ]]; then + echo "*failed*" | tee -a "$LOGFILE" + cleanup 2 + fi + export VCXSRV_PID=$(echo "$TASKS"| grep vcxsrv | awk '{print $2}' 2> /dev/null) + else + export VCXSRV_PID=$(tasklist | grep vcxsrv | awk '{print $2}' 2> /dev/null) + fi + if [ -n "${VCXSRV_PID}" ]; then echo -e "\nVcXsrv running with PID $VCXSRV_PID" >> "$LOGFILE" echo "yes" | tee -a "$LOGFILE" - echo "WARNING: continuing with existing VcXsrv process. You many need to stop it and let this script start VcXsrv for the GUI to display" + if [[ $TRAVIS_TESTING -eq 1 ]]; then + echo -e "\nError: Travis expects that VcXsrv will be started every time" + cleanup 1 + else + echo "WARNING: continuing with existing VcXsrv process. You many need to stop it and let this script start VcXsrv for the GUI to display" + fi else echo "no" | tee -a "$LOGFILE" export VCXSRV_PID= @@ -2078,7 +2436,7 @@ function find_command_suggested_path { __program="$1" __path="$2" - if [[ "$OS" =~ "windows" ]]; then + if [[ "$OS" == "windows" ]] || [[ "$OS" == "wsl" ]]; then __executable_name="$__program.exe" else __executable_name="$__program" diff --git a/scripts/run-travis-windows.sh b/scripts/run-travis-windows.sh index b3a42ea47..8bdcf0100 100755 --- a/scripts/run-travis-windows.sh +++ b/scripts/run-travis-windows.sh @@ -29,4 +29,4 @@ export -f cleanup cd "$DIR/../" export PATH=$PATH:/C/tools/msys64/mingw64/bin -USE_SSH=0 "./hnn_docker.sh" start || script_fail \ No newline at end of file +USE_SSH=0 ./hnn_docker.sh start || script_fail diff --git a/scripts/run-travis-wsl.sh b/scripts/run-travis-wsl.sh new file mode 100755 index 000000000..642d76ed1 --- /dev/null +++ b/scripts/run-travis-wsl.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +export TRAVIS_TESTING=1 + +source "$DIR/../installer/docker/hnn_envs" + +echo "Testing GUI on WSL..." +cd $DIR/../ +export DISPLAY=:0 +python3 hnn.py + +echo "Building HNN in WSL..." +make clean +make -j2 + +echo "Running Python tests in WSL..." +py.test tests/ + +echo "Testing MPI in WSL..." +mpiexec -np 2 nrniv -mpi -python run.py \ No newline at end of file diff --git a/scripts/setup-travis-mac.sh b/scripts/setup-travis-mac.sh index dc4803c0b..f8c24fcda 100755 --- a/scripts/setup-travis-mac.sh +++ b/scripts/setup-travis-mac.sh @@ -1,12 +1,12 @@ #!/bin/bash -set -xe +set -e export TRAVIS_TESTING=1 source scripts/utils.sh export -f sha256sum -export DOCKER_IMAGE_NAME="jonescompneurolab/hnn:master" +export DOCKER_IMAGE_NAME="jonescompneurolab/hnn:latest" export BASE_QEMU_OPTS="--disable-cocoa --disable-curses --disable-vnc --disable-vde \ --disable-pie --disable-libusb --disable-hax --disable-kvm \ --disable-debug-info --disable-docs --disable-nettle \ @@ -175,17 +175,17 @@ start_download "$FILENAME" "$URL" & NRN_PID=$! if [[ $HOMEBREW_QEMU -eq 1 ]] || [[ $BUILD_QEMU -eq 1 ]]; then -echo "Waiting for qemu build to finish" -NAME="building qemu" -wait_for_pid "${MAKE_PID}" "$NAME" + echo "Waiting for qemu build to finish" + NAME="building qemu" + wait_for_pid "${MAKE_PID}" "$NAME" -make install -command cd ${TRAVIS_BUILD_DIR} + make install + command cd ${TRAVIS_BUILD_DIR} fi # create default VM with docker-machine echo "Starting qemu VM..." -docker-machine -D create --driver qemu --qemu-cache-mode unsafe --qemu-cpu-count 2 default & +docker-machine -D create --driver qemu --qemu-cache-mode unsafe --qemu-cpu-count 2 default &> machine.log & export MACHINE_PID=$! curl -Lo "$HOME/download-frozen-image-v2.sh" https://raw.githubusercontent.com/moby/moby/master/contrib/download-frozen-image-v2.sh @@ -211,7 +211,17 @@ conda install -y -n hnn -c conda-forge nlopt pip download flake8 pytest pytest-cov coverage coveralls mne echo "Waiting for VM to start..." -wait $MACHINE_PID +wait $MACHINE_PID || COMMAND_STATUS=$? +if [[ $COMMAND_STATUS -ne 0 ]]; then + echo -e "\n********** Docker machine failed with status $COMMAND_STATUS. Log below *************\n" + cat machine.log + if [ -f /tmp/qemu.log ]; then + echo -e "\n********** QEMU log *************\n" + cat /tmp/qemu.log + fi + echo "Trying to start VM again..." + docker-machine -D create --driver qemu --qemu-cache-mode unsafe --qemu-cpu-count 2 default +fi echo "VM running docker is up" # set up environment variables to use docker within VM @@ -223,9 +233,10 @@ NAME="downloading HNN docker image" wait_for_pid "${IMAGE_PID}" "$NAME" echo "Loading downloaded image into docker" (tar -cC "$HOME/docker_image" . | docker load && \ -docker tag jonescompneurolab/hnn:master jonescompneurolab/hnn:latest && \ touch $HOME/docker_image_loaded) & +# docker tag jonescompneurolab/hnn:latest jonescompneurolab/hnn:latest && \ + # hack so that NEURON install doesn't take forever sudo kill -9 $NRN_INSTALL_PID && wait $NRN_INSTALL_PID || { if [[ $? -eq 137 ]]; then @@ -235,4 +246,4 @@ else fi } -echo "Install finished" \ No newline at end of file +echo "Install finished" diff --git a/scripts/setup-travis-windows.sh b/scripts/setup-travis-windows.sh index 80c1da0d7..625f3ea6c 100755 --- a/scripts/setup-travis-windows.sh +++ b/scripts/setup-travis-windows.sh @@ -8,7 +8,8 @@ source "$DIR/utils.sh" # we use find_program_print and retry_docker_pull from docker_functions.sh source "$DIR/docker_functions.sh" -export LOGFILE="docker_setup.log" + +[[ $LOGFILE ]] || LOGFILE="hnn_travis.log" # docker_functions.sh expects some globals to be set set_globals @@ -32,12 +33,17 @@ function cleanup { } export -f cleanup -# start the docker pull in the background -find_program_print docker -(retry_docker_pull && touch $HOME/docker_image_loaded) & +echo "Installing Ubuntu WSL..." +powershell.exe -ExecutionPolicy Bypass -File ./scripts/setup-travis-wsl.ps1 & +WSL_PID=$! + +# prepare for installing msys2 +[[ ! -f C:/tools/msys64/msys2_shell.cmd ]] && rm -rf C:/tools/msys64 +choco uninstall -y mingw # enable windows remoting service to log in as a different user to run tests powershell -Command 'Start-Service -Name WinRM' > /dev/null +powershell -Command 'Start-Service -Name seclogon' > /dev/null # change settings to allow a blank password for TEST_USER reg add HKEY_LOCAL_MACHINE\\SYSTEM\\CurrentControlSet\\Control\\Lsa //t REG_DWORD //v LimitBlankPasswordUse //d 0 //f 2>&1 > /dev/null @@ -58,51 +64,56 @@ runas //user:"$TEST_USER" "cmd /C whoami" > /dev/null < "$HOME/test_user_creds" # copy hnn source to test user's home directory TEST_USER_DIR="/c/Users/$TEST_USER" if [ -d "$TEST_USER_DIR" ]; then - mkdir "$TEST_USER_DIR/hnn" cp -r "$(pwd)" "$TEST_USER_DIR/" else echo "No user home directory created at $TEST_USER_DIR" exit 2 fi -echo "Starting Windows install script..." -powershell.exe -ExecutionPolicy Bypass -File ./installer/windows/hnn.ps1 & -POWERSHELL_PID=$! +echo "Installing Microsoft MPI" +powershell -command "(New-Object System.Net.WebClient).DownloadFile('https://github.com/microsoft/Microsoft-MPI/releases/download/v10.1.1/msmpisetup.exe', 'msmpisetup.exe')" && \ + ./msmpisetup.exe -unattend && \ + rm -f msmpisetup.exe -echo "Waiting for Windows install script to finish..." -NAME="installing HNN on host system" -wait_for_pid "${POWERSHELL_PID}" "$NAME" +echo "Running HNN Windows install script..." +powershell.exe -ExecutionPolicy Bypass -File ./installer/windows/hnn-windows.ps1 +# add miniconda python to the path +export PATH=$PATH:$HOME/Miniconda3/Scripts +export PATH=$HOME/Miniconda3/envs/hnn/:$PATH +export PATH=$HOME/Miniconda3/envs/hnn/Scripts:$PATH +export PATH=$HOME/Miniconda3/envs/hnn/Library/bin:$PATH -URL="https://downloads.sourceforge.net/project/vcxsrv/vcxsrv/1.20.8.1/vcxsrv-64.1.20.8.1.installer.exe" -FILENAME="$HOME/vcxsrv-64.1.20.8.1.installer.exe" -start_download "$FILENAME" "$URL" > /dev/null & -VCXSRV_PID=$! - -# install msys2 to get opengl32.dll from mesa -# this is needed to be able to start vcxsrv for docker tests -[[ ! -f C:/tools/msys64/msys2_shell.cmd ]] && rm -rf C:/tools/msys64 -choco uninstall -y mingw +echo "Installing msys2 with choco..." +choco upgrade --no-progress -y msys2 &> /dev/null -# install vcxsrv for docker tests -echo "Waiting for VcXsrv download to finish..." -NAME="downloading VcXsrv" -wait_for_pid "${VCXSRV_PID}" "$NAME" +# start the docker pull in the background +echo "Starting to pull the docker image..." +find_program_print docker +((retry_docker_pull > /dev/null || script_fail) && touch $HOME/docker_image_loaded ) & -echo "Installing msys2 with choco..." -choco upgrade --no-progress -y msys2 & -MSYS2_PID=$! +echo "Downloading VcXsrv..." +URL="https://downloads.sourceforge.net/project/vcxsrv/vcxsrv/1.20.8.1/vcxsrv-64.1.20.8.1.installer.exe" +FILENAME="$HOME/vcxsrv-64.1.20.8.1.installer.exe" +start_download "$FILENAME" "$URL" > /dev/null echo "Installing VcXsrv..." cmd //c "$HOME/vcxsrv-64.1.20.8.1.installer.exe /S" -echo "Waiting for msys2 installation to finish..." -NAME="isntalling msys2" -wait_for_pid "${MSYS2_PID}" "$NAME" - +# get opengl32.dll from mesa +# this is needed to be able to start vcxsrv for docker tests export msys2='cmd //C RefreshEnv.cmd ' export msys2+='& set MSYS=winsymlinks:nativestrict ' export msys2+='& C:\\tools\\msys64\\msys2_shell.cmd -defterm -no-start' export mingw64="$msys2 -mingw64 -full-path -here -c "\"\$@"\" --" export msys2+=" -msys2 -c "\"\$@"\" --" -$msys2 pacman --sync --noconfirm --needed mingw-w64-x86_64-mesa & -# the command above will complete before the docker test begins +$msys2 pacman --sync --noconfirm --needed mingw-w64-x86_64-mesa + +echo "Downloading python test packages..." +pip download flake8 pytest pytest-cov coverage coveralls mne + +echo "Downloading python test packages in WSL (command may fail)..." +wsl -- pip download flake8 pytest pytest-cov coverage coveralls mne || true + +echo "Waiting for WSL install to finish..." +NAME="installing WSL" +wait_for_pid "${WSL_PID}" "$NAME" || script_fail diff --git a/scripts/setup-travis-wsl.ps1 b/scripts/setup-travis-wsl.ps1 new file mode 100644 index 000000000..cd0d4a2f7 --- /dev/null +++ b/scripts/setup-travis-wsl.ps1 @@ -0,0 +1,36 @@ +$ErrorActionPreference = "Stop" + +Set-Location C:\Users\travis +Enable-WindowsOptionalFeature -Online -FeatureName Microsoft-Windows-Subsystem-Linux +Write-Host "Downloading Ubuntu image..." +Invoke-WebRequest -Uri https://aka.ms/wsl-ubuntu-1804 -OutFile Ubuntu.appx -UseBasicParsing + +Write-Host "Finished downloading Ubuntu image. Extracting..." +Move-Item .\Ubuntu.appx .\Ubuntu.zip +Expand-Archive .\Ubuntu.zip .\Ubuntu +$userenv = [System.Environment]::GetEnvironmentVariable("Path", "User"); [System.Environment]::SetEnvironmentVariable("PATH", $userenv + ";C:\Users\travis\Ubuntu", "User") + +Write-Host "Configuring Ubuntu WSL..." +& .\Ubuntu\Ubuntu1804.exe install --root + +# add hnn_user +& wsl -- bash -ec "groupadd hnn_group && useradd -m -b /home/ -g hnn_group hnn_user && adduser hnn_user sudo && echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers && chsh -s /bin/bash hnn_user" + +# copy hnn dir to hnn_user homedir and change permissions +& wsl -- bash -ec "cp -r build/jonescompneurolab/hnn /home/hnn_user/ && chown -R hnn_user: /home/hnn_user && apt-get update && apt-get install -y dos2unix" + +# run future commands as hnn_user +& .\Ubuntu\Ubuntu1804.exe config --default-user hnn_user + +# remove windows newlines +& wsl -- bash -ec "dos2unix /home/hnn_user/hnn/scripts/* /home/hnn_user/hnn/installer/ubuntu/hnn-ubuntu.sh /home/hnn_user/hnn/installer/docker/hnn_envs" + +Write-Host "Installing HNN in Ubuntu WSL..." +& wsl -- bash -ec "cd /home/hnn_user/hnn && source scripts/utils.sh && export LOGFILE=ubuntu_install.log && TRAVIS_TESTING=1 installer/ubuntu/hnn-ubuntu.sh || script_fail" + +if (!$?) { + exit 1 +} +else { + exit 0 +} diff --git a/scripts/start-test-as-user.ps1 b/scripts/start-test-as-user.ps1 index 76aa731d9..cd03f4b54 100644 --- a/scripts/start-test-as-user.ps1 +++ b/scripts/start-test-as-user.ps1 @@ -1,9 +1,9 @@ $ErrorActionPreference = "Stop" ( Enable-PSRemoting -Force -SkipNetworkProfileCheck ) | out-null -winrm set winrm/config/service '@{AllowUnencrypted="true"}' -winrm set winrm/config/service/auth '@{Basic="true"}' -winrm set winrm/config/client '@{AllowUnencrypted="true"}' +(winrm set winrm/config/service '@{AllowUnencrypted="true"}' ) | out-null +(winrm set winrm/config/service/auth '@{Basic="true"}' ) | out-null +(winrm set winrm/config/client '@{AllowUnencrypted="true"}' ) | out-null # create credentials with no password ( $Credentials = [System.Management.Automation.PSCredential]::new("test user",[System.Security.SecureString]::new()) ) | out-null @@ -15,19 +15,31 @@ winrm set winrm/config/client '@{AllowUnencrypted="true"}' $session = New-PSSession -ComputerName localhost -SessionOption $opt -Authentication Basic -Credential $Credentials # Run the bash command as 'test user' -$env:ret = $false -Write-Output "Starting run-travis-windows.sh as user 'test user'" -Invoke-Command -Session $session -ScriptBlock { - $script:run_script="/c/users/$env:UserName/hnn/scripts/run-travis-windows.sh" - & "C:\Program Files\Git\bin\bash.exe" "$script:run_script" - Write-Output "Return status: $?" - $env:ret=$? +Write-Output "Starting run-travis-windows.sh as user 'test user'..." +try { + Invoke-Command -Session $session -ScriptBlock { + $script:run_script="/c/users/$env:UserName/hnn/scripts/run-travis-windows.sh" + try { + & "C:\Program Files\Git\bin\bash.exe" "$script:run_script" + if (!$?) { + Write-Output "run-travis-windows.sh returned $LastExitCode" + $host.SetShouldExit($LastExitCode) + exit $LastExitCode + } + } + catch { + Write-Output "Failed to run run-travis-windows.sh" + Write-Output $_ + $host.SetShouldExit(-1) + throw + } + } } - -if (!$env:ret) { +catch { + Write-Output "Failed to start session:" + Write-Output $_ $host.SetShouldExit(-1) - throw + exit -1 } -else { - exit 0 -} \ No newline at end of file + +Remove-PSSession -Session $session \ No newline at end of file diff --git a/scripts/utils.sh b/scripts/utils.sh index a82113c20..1bcf6c7f3 100755 --- a/scripts/utils.sh +++ b/scripts/utils.sh @@ -50,11 +50,10 @@ function wait_for_pid { } function script_fail { - # This function is specific to .travis.yml. always use hnn_docker.log and not - # $LOGFILE + check_var LOGFILE echo -ne "\n******* hnn_docker.sh failed. output from hnn_docker.log below *******\n" - cat hnn_docker.log + cat "$LOGFILE" exit 2 }