Skip to content

Commit

Permalink
tests: add tests for setup-env.sh
Browse files Browse the repository at this point in the history
- tests use a shell-script harness and test all Spack commands that
  require special shell support.

- tests work in bash, zsh, and dash

- run setup-env.sh tests on macos and linux builds.
  - we run them on macos and linux
  • Loading branch information
tgamblin committed Jul 3, 2019
1 parent ff02c61 commit 52c936e
Show file tree
Hide file tree
Showing 4 changed files with 325 additions and 27 deletions.
33 changes: 10 additions & 23 deletions .travis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -50,24 +50,6 @@ jobs:
os: linux
language: python
env: [ TEST_SUITE=unit, COVERAGE=true ]
addons:
apt:
packages:
- cmake
- gfortran
- graphviz
- gnupg2
- kcov
- mercurial
- ninja-build
- perl
- perl-base
- realpath
- patchelf
- r-base
- r-base-core
- r-base-dev

- python: '3.7'
sudo: required
os: linux
Expand Down Expand Up @@ -159,13 +141,16 @@ addons:
- r-base
- r-base-core
- r-base-dev
- zsh
# for Mac builds, we use Homebrew
homebrew:
packages:
- python@2
- gcc
- gnupg2
- ccache
- dash
- kcov
update: true

# ~/.ccache needs to be cached directly as Travis is not taking care of it
Expand Down Expand Up @@ -223,11 +208,13 @@ script:

after_success:
- ccache -s
- if [[ "$TEST_SUITE" == "unit" || "$TEST_SUITE" == "build" ]]; then
codecov --env PYTHON_VERSION
--required
--flags "${TEST_SUITE}${TRAVIS_OS_NAME}";
fi
- case "$TEST_SUITE" in
unit)
codecov --env PYTHON_VERSION
--required
--flags "${TEST_SUITE}${TRAVIS_OS_NAME}";
;;
esac

#=============================================================================
# Notifications
Expand Down
28 changes: 28 additions & 0 deletions share/spack/qa/run-unit-tests
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,12 @@
# Optionally add one or more unit tests
# to only run these tests.
#

#-----------------------------------------------------------
# Run a few initial commands and set up test environment
#-----------------------------------------------------------
ORIGINAL_PATH="$PATH"

. "$(dirname $0)/setup.sh"
check_dependencies ${coverage} git hg svn

Expand All @@ -33,9 +39,31 @@ bin/spack help -a
# Profile and print top 20 lines for a simple call to spack spec
bin/spack -p --lines 20 spec mpileaks%gcc ^elfutils@0.170

#-----------------------------------------------------------
# Run unit tests with code coverage
#-----------------------------------------------------------
extra_args=""
if [[ -n "$@" ]]; then
extra_args="-k $@"
fi
${coverage_run} bin/spack test --verbose "$extra_args"

#-----------------------------------------------------------
# Run tests for setup-env.sh
#-----------------------------------------------------------
# Clean the environment by removing Spack from the path and getting rid of
# the spack shell function
export PATH="$ORIGINAL_PATH"
unset spack

# start in the spack root directory
cd $SPACK_ROOT

# Run bash tests with coverage enabled, but pipe output to /dev/null
# because it seems that kcov seems to undo the script's redirection
${QA_DIR}/bashcov ${QA_DIR}/setup-env-test.sh &> /dev/null

# run the test scripts for their output (these will print nicely)
bash ${QA_DIR}/setup-env-test.sh
zsh ${QA_DIR}/setup-env-test.sh
dash ${QA_DIR}/setup-env-test.sh
286 changes: 286 additions & 0 deletions share/spack/qa/setup-env-test.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,286 @@
#!/bin/sh
#
# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

#
# This script tests that Spack's setup-env.sh init script works.
#
# The tests are portable to bash, zsh, and bourne shell, and can be run
# in any of these shells.
#

# ------------------------------------------------------------------------
# Functions for color output.
# ------------------------------------------------------------------------

# Colors for output
red='\033[1;31m'
cyan='\033[1;36m'
green='\033[1;32m'
reset='\033[0m'

echo_red() {
printf "${red}$*${reset}\n"
}

echo_green() {
printf "${green}$*${reset}\n"
}

echo_msg() {
printf "${cyan}$*${reset}\n"
}

# ------------------------------------------------------------------------
# Generic functions for testing shell code.
# ------------------------------------------------------------------------

# counts of test successes and failures.
success=0
errors=0

# Print out a header for a group of tests.
title() {
echo
echo_msg "$@"
echo_msg "---------------------------------"
}

# echo FAIL in red text; increment failures
fail() {
echo_red FAIL
errors=$((errors+1))
}

#
# Echo SUCCESS in green; increment successes
#
pass() {
echo_green SUCCESS
success=$((success+1))
}

#
# Run a command and suppress output unless it fails.
# On failure, echo the exit code and output.
#
succeeds() {
printf "'%s' succeeds ... " "$*"
output=$("$@" 2>&1)
err="$?"

if [ "$err" != 0 ]; then
fail
echo_red "Command failed with error $err."
if [ -n "$output" ]; then
echo_msg "Output:"
echo "$output"
else
echo_msg "No output."
fi
else
pass
fi
}

#
# Run a command and suppress output unless it succeeds.
# If the command succeeds, echo the output.
#
fails() {
printf "'%s' fails ... " "$*"
output=$("$@" 2>&1)
err="$?"

if [ "$err" = 0 ]; then
fail
echo_red "Command failed with error $err."
if [ -n "$output" ]; then
echo_msg "Output:"
echo "$output"
else
echo_msg "No output."
fi
else
pass
fi
}

#
# Ensure that a string is in the output of a command.
# Suppresses output on success.
# On failure, echo the exit code and output.
#
contains() {
string="$1"
shift

printf "'%s' output contains '$string' ... " "$*"
output=$("$@" 2>&1)
err="$?"

if [ "${output#*$string}" = "${output}" ]; then
fail
echo_red "Command exited with error $err."
echo_red "'$string' was not in output."
if [ -n "$output" ]; then
echo_msg "Output:"
echo "$output"
else
echo_msg "No output."
fi
else
pass
fi
}

# -----------------------------------------------------------------------
# Mock module and dotkit commands. These just print what they would do,
# so that we can test their output.
# -----------------------------------------------------------------------
module() {
echo module "$@"
}

use() {
echo use "$@"
}

unuse() {
echo unuse "$@"
}

# -----------------------------------------------------------------------
# Setup test environment and do some preliminary checks
# -----------------------------------------------------------------------

# Make sure no environment is active
unset SPACK_ENV

# Source setup-env.sh before tests
. share/spack/setup-env.sh

title "Testing setup-env.sh with $_sp_shell"

# spack command is now avaialble
succeeds which spack

# mock cd command (intentionally define only AFTER setup-env.sh)
cd() {
echo cd "$@"
}

# create a fake mock package install and store its location for later
title "Setup"
echo "Creating a mock package installation"
spack -m install --fake a
a_install=$(spack location -i a)
a_module=$(spack -m module tcl find a)
a_dotkit=$(spack -m module dotkit find a)

b_install=$(spack location -i b)
b_module=$(spack -m module tcl find b)
b_dotkit=$(spack -m module dotkit find b)

# ensure that we uninstall b on exit
cleanup() {
title "Cleanup"
echo "Removing test package before exiting test script."
spack -m uninstall -yf b
spack -m uninstall -yf a

echo
echo "$success tests succeeded."
echo "$errors tests failed."
if [ "$errors" = 0 ]; then
pass
exit 0
else
fail
exit 1
fi
}
trap cleanup EXIT

# -----------------------------------------------------------------------
# Test all spack commands with special env support
# -----------------------------------------------------------------------
title 'Testing `spack`'
contains 'usage: spack ' spack
contains "usage: spack " spack -h
contains "usage: spack " spack help
contains "usage: spack " spack -H
contains "usage: spack " spack help --all

title 'Testing `spack cd`'
contains "usage: spack cd " spack cd -h
contains "usage: spack cd " spack cd --help
contains "cd $b_install" spack cd -i b

title 'Testing `spack module`'
contains "usage: spack module " spack -m module -h
contains "usage: spack module " spack -m module --help
contains "usage: spack module " spack -m module

title 'Testing `spack load`'
contains "module load $b_module" spack -m load b
fails spack -m load -l
contains "module load -l --arg $b_module" spack -m load -l --arg b
contains "module load $b_module $a_module" spack -m load -r a
contains "module load $b_module $a_module" spack -m load --dependencies a
fails spack -m load d
contains "usage: spack load " spack -m load -h
contains "usage: spack load " spack -m load -h d
contains "usage: spack load " spack -m load --help

title 'Testing `spack unload`'
contains "module unload $b_module" spack -m unload b
fails spack -m unload -l
contains "module unload -l --arg $b_module" spack -m unload -l --arg b
fails spack -m unload d
contains "usage: spack unload " spack -m unload -h
contains "usage: spack unload " spack -m unload -h d
contains "usage: spack unload " spack -m unload --help

title 'Testing `spack use`'
contains "use $b_dotkit" spack -m use b
fails spack -m use -l
contains "use -l --arg $b_dotkit" spack -m use -l --arg b
contains "use $b_dotkit $a_dotkit" spack -m use -r a
contains "use $b_dotkit $a_dotkit" spack -m use --dependencies a
fails spack -m use d
contains "usage: spack use " spack -m use -h
contains "usage: spack use " spack -m use -h d
contains "usage: spack use " spack -m use --help

title 'Testing `spack unuse`'
contains "unuse $b_dotkit" spack -m unuse b
fails spack -m unuse -l
contains "unuse -l --arg $b_dotkit" spack -m unuse -l --arg b
fails spack -m unuse d
contains "usage: spack unuse " spack -m unuse -h
contains "usage: spack unuse " spack -m unuse -h d
contains "usage: spack unuse " spack -m unuse --help

title 'Testing `spack env`'
contains "usage: spack env " spack env -h
contains "usage: spack env " spack env --help

title 'Testing `spack env activate`'
contains "No such environment:" spack env activate no_such_environment
contains "usage: spack env activate " spack env activate
contains "usage: spack env activate " spack env activate -h
contains "usage: spack env activate " spack env activate --help

title 'Testing `spack env deactivate`'
contains "Error: No environment is currently active" spack env deactivate
contains "usage: spack env deactivate " spack env deactivate no_such_environment
contains "usage: spack env deactivate " spack env deactivate -h
contains "usage: spack env deactivate " spack env deactivate --help

title 'Testing `spack env list`'
contains " spack env list " spack env list -h
contains " spack env list " spack env list --help
Loading

0 comments on commit 52c936e

Please sign in to comment.