Skip to content

Commit

Permalink
Merge pull request #3228 from edsantiago/bats_assert
Browse files Browse the repository at this point in the history
tests: new assert()
  • Loading branch information
openshift-merge-robot committed Jul 30, 2021
2 parents f517d85 + e62277e commit 56ff12f
Show file tree
Hide file tree
Showing 7 changed files with 224 additions and 96 deletions.
29 changes: 13 additions & 16 deletions tests/bud.bats
Expand Up @@ -1510,7 +1510,7 @@ function _test_http() {
run_buildah bud --signature-policy ${TESTSDIR}/policy.json -t ${target} --build-arg foo=bar --build-arg foo2=bar2 -f ${TESTSDIR}/bud/build-arg ${TESTSDIR}/bud/build-arg
expect_output --substring "one or more build args were not consumed: \[foo2\]"
run_buildah bud --signature-policy ${TESTSDIR}/policy.json -t ${target} --build-arg IMAGE=alpine -f ${TESTSDIR}/bud/build-arg/Dockerfile2 ${TESTSDIR}/bud/build-arg
! expect_output --substring "one or more build args were not consumed: \[IMAGE\]"
assert "$output" !~ "one or more build args were not consumed: \[IMAGE\]"
expect_output --substring "FROM alpine"
}

Expand Down Expand Up @@ -1807,7 +1807,7 @@ _EOF
run_buildah bud -t testbud --signature-policy ${TESTSDIR}/policy.json ${mytmpdir}
expect_output --substring "file1"
expect_output --substring "file2"
! expect_output --substring "file3"
assert "$output" !~ "file3"
}

@test "bud copy with .dockerignore #2" {
Expand All @@ -1827,8 +1827,8 @@ RUN find /tmp/stuff -type f
_EOF

run_buildah bud -t testbud --signature-policy ${TESTSDIR}/policy.json ${mytmpdir}
! expect_output --substring "file1"
! expect_output --substring "file2"
assert "$output" !~ file1
assert "$output" !~ file2
}

@test "bud-copy-workdir" {
Expand Down Expand Up @@ -1869,9 +1869,8 @@ _EOF
run_buildah bud --signature-policy ${TESTSDIR}/policy.json --layers -t ${target} -f Dockerfile3 --build-arg=UID=17122 --build-arg=CODE=/copr/coprs_frontend --build-arg=USERNAME=praiskup --build-arg=PGDATA=/pgdata ${TESTSDIR}/bud/build-arg
run_buildah inspect -f '{{.FromImageID}}' ${target}
argsid="$output"
if [[ "$argsid" == "$initialid" ]]; then
die ".FromImageID of test-img-2 ($argsid) == same as test-img, it should be different"
fi
assert "$argsid" != "$initialid" \
".FromImageID of test-img-2 ($argsid) == same as test-img, it should be different"

# With build args, even in a different order, we should end up using the previous build as a cached result.
run_buildah bud --signature-policy ${TESTSDIR}/policy.json --layers -t ${target} -f Dockerfile3 --build-arg=UID=17122 --build-arg=CODE=/copr/coprs_frontend --build-arg=USERNAME=praiskup --build-arg=PGDATA=/pgdata ${TESTSDIR}/bud/build-arg
Expand Down Expand Up @@ -1905,18 +1904,16 @@ _EOF
run_buildah bud --signature-policy ${TESTSDIR}/policy.json --layers -t test-img-2 --build-arg TEST=foo -f Dockerfile4 ${TESTSDIR}/bud/build-arg
run_buildah inspect -f '{{.FromImageID}}' test-img-2
argsid="$output"
if [[ "$argsid" == "$initialid" ]]; then
die ".FromImageID of test-img-2 ($argsid) == same as test-img, it should be different"
fi
assert "$argsid" != "$initialid" \
".FromImageID of test-img-2 ($argsid) == same as test-img, it should be different"

# Set the build-arg via an ENV in the local environment and verify that the cached layers are not used
export TEST=bar
run_buildah bud --signature-policy ${TESTSDIR}/policy.json --layers -t test-img-3 --build-arg TEST -f Dockerfile4 ${TESTSDIR}/bud/build-arg
run_buildah inspect -f '{{.FromImageID}}' test-img-3
argsid="$output"
if [[ "$argsid" == "$initialid" ]]; then
die ".FromImageID of test-img-3 ($argsid) == same as test-img, it should be different"
fi
assert "$argsid" != "$initialid" \
".FromImageID of test-img-3 ($argsid) == same as test-img, it should be different"
}

@test "bud test RUN with a privileged command" {
Expand Down Expand Up @@ -2869,12 +2866,12 @@ RUN echo "$SECRET"
_EOF

run_buildah bud -t testbud --signature-policy ${TESTSDIR}/policy.json --file ${mytmpdir} .
! expect_output --substring '\-\-build-arg SECRET=<VALUE>'
assert "$output" !~ '--build-arg SECRET=<VALUE>'
expect_output --substring '\-\-build-arg NEWSECRET=<VALUE>'

run_buildah bud -t testbud --signature-policy ${TESTSDIR}/policy.json --build-arg NEWSECRET="VerySecret" --file ${mytmpdir} .
! expect_output --substring '\-\-build-arg SECRET=<VALUE>'
! expect_output --substring '\-\-build-arg NEWSECRET=<VALUE>'
assert "$output" !~ '--build-arg SECRET=<VALUE>'
assert "$output" !~ '--build-arg NEWSECRET=<VALUE>'
}

@test "bud with --runtime and --runtime-flag" {
Expand Down
6 changes: 3 additions & 3 deletions tests/config.bats
Expand Up @@ -23,19 +23,19 @@ load helpers
run_buildah config --annotation ANNOTATION $cid

run_buildah 125 config --healthcheck 'AB "CD' $cid
expect_output --substring 'error parsing --healthcheck "AB \"CD": invalid command line string'
expect_output --substring 'error parsing --healthcheck "AB \\"CD": invalid command line string'

run_buildah 125 config --healthcheck-interval ABCD $cid
expect_output --substring 'error parsing --healthcheck-interval "ABCD": time: invalid duration "?ABCD"?'

run_buildah 125 config --cmd 'AB "CD' $cid
expect_output --substring 'error parsing --cmd "AB \"CD": invalid command line string'
expect_output --substring 'error parsing --cmd "AB \\"CD": invalid command line string'

run_buildah 125 config --env ENV $cid
expect_output --substring 'error setting env "ENV": no value given'

run_buildah 125 config --shell 'AB "CD' $cid
expect_output --substring 'error parsing --shell "AB \"CD": invalid command line string'
expect_output --substring 'error parsing --shell "AB \\"CD": invalid command line string'
}

function check_matrix() {
Expand Down
171 changes: 112 additions & 59 deletions tests/helpers.bash
Expand Up @@ -19,17 +19,23 @@ export GPG_TTY=/dev/null
function setup() {
pushd "$(dirname "$(readlink -f "$BASH_SOURCE")")"

suffix=$(dd if=/dev/urandom bs=12 count=1 status=none | od -An -tx1 | sed -e 's, ,,g')
TESTDIR=${BATS_TMPDIR}/tmp${suffix}
rm -fr ${TESTDIR}
mkdir -p ${TESTDIR}/{root,runroot,sigstore,registries.d,cache}
echo "default-docker: " >> ${TESTDIR}/registries.d/default.yaml
echo " sigstore-staging: file://${TESTDIR}/sigstore " >> ${TESTDIR}/registries.d/default.yaml
echo "docker: " >> ${TESTDIR}/registries.d/default.yaml
echo " registry.access.redhat.com: " >> ${TESTDIR}/registries.d/default.yaml
echo " sigstore: https://access.redhat.com/webassets/docker/content/sigstore " >> ${TESTDIR}/registries.d/default.yaml
echo " registry.redhat.io: " >> ${TESTDIR}/registries.d/default.yaml
echo " sigstore: https://registry.redhat.io/containers/sigstore " >> ${TESTDIR}/registries.d/default.yaml
# buildah/podman: "repository name must be lowercase".
# me: "but it's a local file path, not a repository name!"
# buildah/podman: "i dont care. no caps anywhere!"
TESTDIR=$(mktemp -d --dry-run --tmpdir=${BATS_TMPDIR:-${TMPDIR:-/tmp}} buildah_tests.XXXXXX | tr A-Z a-z)
mkdir --mode=0700 $TESTDIR

mkdir -p ${TESTDIR}/{root,runroot,sigstore,registries.d}
cat >${TESTDIR}/registries.d/default.yaml <<EOF
default-docker:
sigstore-staging: file://${TESTDIR}/sigstore
docker:
registry.access.redhat.com:
sigstore: https://access.redhat.com/webassets/docker/content/sigstore
registry.redhat.io:
sigstore: https://registry.redhat.io/containers/sigstore
EOF

# Common options for all buildah and podman invocations
ROOTDIR_OPTS="--root ${TESTDIR}/root --runroot ${TESTDIR}/runroot --storage-driver ${STORAGE_DRIVER}"
BUILDAH_REGISTRY_OPTS="--registries-conf ${TESTSDIR}/registries.conf --registries-conf-dir ${TESTDIR}/registries.d --short-name-alias-conf ${TESTDIR}/cache/shortnames.conf"
Expand Down Expand Up @@ -222,77 +228,124 @@ function die() {
false
}

###################
# expect_output # Compare actual vs expected string; fail if mismatch
###################
############
# assert # Compare actual vs expected string; fail if mismatch
############
#
# Compares $output against the given string argument. Optional second
# argument is descriptive text to show as the error message (default:
# the command most recently run by 'run_buildah'). This text can be
# useful to isolate a failure when there are multiple identical
# run_buildah invocations, and the difference is solely in the
# config or setup; see, e.g., run.bats:run-cmd().
# Compares string (default: $output) against the given string argument.
# By default we do an exact-match comparison against $output, but there
# are two different ways to invoke us, each with an optional description:
#
# By default we run an exact string comparison; use --substring to
# look for the given string anywhere in $output.
# xpect "EXPECT" [DESCRIPTION]
# xpect "RESULT" "OP" "EXPECT" [DESCRIPTION]
#
# By default we look in "$output", which is set in run_buildah().
# To override, use --from="some-other-string" (e.g. "${lines[0]}")
# The first form (one or two arguments) does an exact-match comparison
# of "$output" against "EXPECT". The second (three or four args) compares
# the first parameter against EXPECT, using the given OPerator. If present,
# DESCRIPTION will be displayed on test failure.
#
# Examples:
#
# expect_output "this is exactly what we expect"
# expect_output "foo=bar" "description of this particular test"
# expect_output --from="${lines[0]}" "expected first line"
# xpect "this is exactly what we expect"
# xpect "${lines[0]}" =~ "^abc" "first line begins with abc"
#
function assert() {
local actual_string="$output"
local operator='=='
local expect_string="$1"
local testname="$2"

case "${#*}" in
0) die "Internal error: 'assert' requires one or more arguments" ;;
1|2) ;;
3|4) actual_string="$1"
operator="$2"
expect_string="$3"
testname="$4"
;;
*) die "Internal error: too many arguments to 'assert" ;;
esac

# Comparisons.
# Special case: there is no !~ operator, so fake it via '! x =~ y'
local not=
local actual_op="$operator"
if [[ $operator == '!~' ]]; then
not='!'
actual_op='=~'
fi
if [[ $operator == '=' || $operator == '==' ]]; then
# Special case: we can't use '=' or '==' inside [[ ... ]] because
# the right-hand side is treated as a pattern... and '[xy]' will
# not compare literally. There seems to be no way to turn that off.
if [ "$actual_string" = "$expect_string" ]; then
return
fi
else
if eval "[[ $not \$actual_string $actual_op \$expect_string ]]"; then
return
elif [ $? -gt 1 ]; then
die "Internal error: could not process 'actual' $operator 'expect'"
fi
fi

# Test has failed. Get a descriptive test name.
if [ -z "$testname" ]; then
testname="${MOST_RECENT_BUILDAH_COMMAND:-[no test name given]}"
fi

# Display optimization: the typical case for 'expect' is an
# exact match ('='), but there are also '=~' or '!~' or '-ge'
# and the like. Omit the '=' but show the others; and always
# align subsequent output lines for ease of comparison.
local op=''
local ws=''
if [ "$operator" != '==' ]; then
op="$operator "
ws=$(printf "%*s" ${#op} "")
fi

# This is a multi-line message, which may in turn contain multi-line
# output, so let's format it ourself, readably
local actual_split
IFS=$'\n' read -rd '' -a actual_split <<<"$actual_string" || true
printf "#/vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv\n" >&2
printf "#| FAIL: %s\n" "$testname" >&2
printf "#| expected: %s'%s'\n" "$op" "$expect_string" >&2
printf "#| actual: %s'%s'\n" "$ws" "${actual_split[0]}" >&2
local line
for line in "${actual_split[@]:1}"; do
printf "#| > %s'%s'\n" "$ws" "$line" >&2
done
printf "#\\^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n" >&2
false
}

###################
# expect_output # [obsolete; kept for compatibility]
###################
#
# An earlier version of assert().
#
function expect_output() {
# By default we examine $output, the result of run_buildah
local actual="$output"
local check_substring=
local operator='=='

# option processing: recognize --from="...", --substring
local opt
for opt; do
local value=$(expr "$opt" : '[^=]*=\(.*\)')
case "$opt" in
--from=*) actual="$value"; shift;;
--substring) check_substring=1; shift;;
--substring) operator='=~'; shift;;
--) shift; break;;
-*) die "Invalid option '$opt'" ;;
*) break;;
esac
done

local expect="$1"
local testname="${2:-${MOST_RECENT_BUILDAH_COMMAND:-[no test name given]}}"

if [ -z "$expect" ]; then
if [ -z "$actual" ]; then
return
fi
expect='[no output]'
elif [ "$actual" = "$expect" ]; then
return
elif [ -n "$check_substring" ]; then
if [[ "$actual" =~ $expect ]]; then
return
fi
fi

# This is a multi-line message, which may in turn contain multi-line
# output, so let's format it ourselves, readably
local -a actual_split
readarray -t actual_split <<<"$actual"
printf "#/vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv\n" >&2
printf "#| FAIL: %s\n" "$testname" >&2
printf "#| expected: '%s'\n" "$expect" >&2
printf "#| actual: '%s'\n" "${actual_split[0]}" >&2
local line
for line in "${actual_split[@]:1}"; do
printf "#| > '%s'\n" "$line" >&2
done
printf "#\\^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n" >&2
false
assert "$actual" "$operator" "$@"
}

#######################
Expand Down
87 changes: 87 additions & 0 deletions tests/helpers.bash.t
@@ -0,0 +1,87 @@
#!/bin/bash
#
# tests for helpers.bash
#

. $(dirname ${BASH_SOURCE})/helpers.bash

INDEX=1
RC=0

# t (true) : tests that should pass
function t() {
result=$(assert "$@" 2>&1)
status=$?

if [[ $status -eq 0 ]]; then
echo "ok $INDEX $*"
else
echo "not ok $INDEX $*"
echo "$result"
RC=1
fi

INDEX=$((INDEX + 1))
}

# f (false) : tests that should fail
function f() {
result=$(assert "$@" 2>&1)
status=$?

if [[ $status -ne 0 ]]; then
echo "ok $INDEX ! $*"
else
echo "not ok $INDEX ! $* [passed, should have failed]"
RC=1
fi

INDEX=$((INDEX + 1))
}



t "" = ""
t "a" != ""
t "" != "a"

t "a" = "a"
t "aa" == "aa"
t "a[b]{c}" = "a[b]{c}"

t "abcde" =~ "a"
t "abcde" =~ "b"
t "abcde" =~ "c"
t "abcde" =~ "d"
t "abcde" =~ "e"
t "abcde" =~ "ab"
t "abcde" =~ "abc"
t "abcde" =~ "abcd"
t "abcde" =~ "bcde"
t "abcde" =~ "cde"
t "abcde" =~ "de"

t "foo" =~ "foo"
t "foobar" =~ "foo"
t "barfoo" =~ "foo"

t 'a "AB \"CD": ef' = 'a "AB \"CD": ef'
t 'a "AB \"CD": ef' =~ 'a "AB \\"CD": ef'

t 'abcdef' !~ 'efg'
t 'abcdef' !~ 'x'

###########

f "a" = "b"
f "a" == "b"

f "abcde" =~ "x"

f "abcde" !~ "a"
f "abcde" !~ "ab"
f "abcde" !~ "abc"

f "" != ""

exit $RC

0 comments on commit 56ff12f

Please sign in to comment.