Merge pull request containers#2533 from edsantiago/bats

New system tests under BATS

test/system/000-TEMPLATE

@@ -0,0 +1,114 @@
+#!/usr/bin/env bats   -*- bats -*-
+#
+# FIXME: short description of the purpose of this module
+#
+# FIXME: copy this file to 'NNN-yourtestname.bats' and edit as needed.
+#
+
+load helpers
+
+@test "podman subcmd - description of this particular test" {
+    args="some sort of argument list"
+    run_podman subcmd $args
+    is "$output"   "what we expect"   "output from 'podman subcmd $args'"
+}
+
+# vim: filetype=sh
+
+###############################################################################
+#
+# FIXME FIXME FIXME: Most of the time you can cut from here on down.
+# FIXME FIXME FIXME: The above template is probably enough for many tests.
+# FIXME FIXME FIXME:
+# FIXME FIXME FIXME: If you need anything more complicated, read on.
+#
+# FIXME: This is a bloated test template. It provides mostly stuff for you
+# FIXME: to remove, plus stuff for you to base your tests on.
+# FIXME:
+# FIXME: copy this file to 'NNN-yourtestname.bats' and edit as needed.
+# FIXME: Read all FIXMEs, act on them as needed, then remove them.
+# FIXME: test w/ $ PODMAN=./bin/podman bats test/system/NNN-yourtestname.bats
+#
+
+load helpers
+
+# FIXME: DELETE THESE LINES UNLESS YOU ABSOLUTELY NEED THEM.
+# FIXME: Most tests will not need a custom setup/teardown: they are
+# FIXME: provided by helpers.bash.
+# FIXME: But if you have to do anything special, these give you the
+# FIXME: names of the standard setup/teardown so you can call them
+# FIXME: before or after your own additions.
+function setup() {
+    basic_setup
+    # FIXME: you almost certainly want to do your own setup _after_ basic.
+}
+function teardown() {
+    # FIXME: you almost certainly want to do your own teardown _before_ basic.
+    basic_teardown
+}
+
+
+# FIXME: very basic one-pass example
+@test "podman FOO - description of test" {
+    # FIXME: please try to remove this line; that is, try to write tests
+    # that will pass as both root and rootless.
+    skip_if_rootless
+
+    # FIXME: template for run commands. Always use 'run_podman'!
+    # FIXME: The '?' means 'ignore exit status'; use a number if you
+    # FIXME:    expect a precise nonzero code, or omit for 0 (usual case).
+    # FIXME: NEVER EVER RUN 'podman' DIRECTLY. See helpers.bash for why.
+    run_podman '?' run -d $IMAGE sh -c 'prep..; echo READY'
+    cid="$output"
+    wait_for_ready $cid
+
+    run_podman logs $cid
+    # FIXME: example of dprint. This will trigger if PODMAN_TEST_DEBUG=FOO
+    # FIXME:  ...or anything that matches the name assigned in the @test line.
+    dprint "podman logs $cid -> '$output'"
+    is "$output" "what are we expecting?" "description of this check"
+
+    # Clean up
+    run_podman rm $cid
+}
+
+
+# FIXME: another example, this time with a test table loop
+@test "podman FOO - json - template for playing with json output" {
+    # FIXME: Define a multiline string in tabular form, using '|' as separator.
+    # FIXME: Each row defines one test. Each column (there may be as many as
+    # FIXME: you want) is one field. In the case below we have two, a
+    # FIXME: json field descriptor and an expected value.
+    tests="
+id        | [0-9a-f]\\\{64\\\}
+created   | [0-9-]\\\+T[0-9:]\\\+\\\.[0-9]\\\+Z
+size      | -\\\?[0-9]\\\+
+"
+
+    # FIXME: Run a basic podman command. We'll check $output multiple times
+    # FIXME: in the while loop below.
+    run_podman history --format json $IMAGE
+
+    # FIXME: parse_table is what does all the work, giving us test cases.
+    parse_table "$tests" | while read field expect; do
+        # FIXME: this shows a drawback of BATS and bash: we can't include '|'
+        # FIXME: in the table, but we need to because some images don't
+        # FIXME: have a CID. So, yeah, this is ugly -- but rare.
+        if [ "$field" = "id" ]; then expect="$expect\|<missing>";fi
+
+        # output is an array of dicts; check each one
+        count=$(echo "$output" | jq '. | length')
+        i=0
+        while [ $i -lt $count ]; do
+            actual=$(echo "$output" | jq -r ".[$i].$field")
+            # FIXME: please be sure to note the third field!
+            # FIXME: that's the test name. Make it something useful! Include
+            # FIXME: loop variables whenever possible. Don't just say "my test"
+            is "$actual"    "$expect\$"    "jq .[$i].$field"
+            i=$(expr $i + 1)
+        done
+    done
+}
+
+
+# vim: filetype=sh

test/system/001-basic.bats

@@ -0,0 +1,50 @@
+#!/usr/bin/env bats
+#
+# Simplest set of podman tests. If any of these fail, we have serious problems.
+#
+
+load helpers
+
+# Override standard setup! We don't yet trust podman-images or podman-rm
+function setup() {
+    :
+}
+
+@test "podman version emits reasonable output" {
+    run_podman version
+
+    is "${lines[0]}" "Version:[ ]\+[1-9][0-9.]\+" "Version line 1"
+    is "$output" ".*Go Version: \+"               "'Go Version' in output"
+    is "$output" ".*RemoteAPI Version: \+"        "API version in output"
+}
+
+
+@test "podman can pull an image" {
+    run_podman pull $IMAGE
+}
+
+# This is for development only; it's intended to make sure our timeout
+# in run_podman continues to work. This test should never run in production
+# because it will, by definition, fail.
+@test "timeout" {
+    if [ -z "$PODMAN_RUN_TIMEOUT_TEST" ]; then
+        skip "define \$PODMAN_RUN_TIMEOUT_TEST to enable this test"
+    fi
+    PODMAN_TIMEOUT=10 run_podman run $IMAGE sleep 90
+    echo "*** SHOULD NEVER GET HERE"
+}
+
+
+# Too many tests rely on jq for parsing JSON.
+#
+# If absolutely necessary, one could establish a convention such as
+# defining PODMAN_TEST_SKIP_JQ=1 and adding a skip_if_no_jq() helper.
+# For now, let's assume this is not absolutely necessary.
+@test "jq is installed and produces reasonable output" {
+    type -path jq >/dev/null || die "FATAL: 'jq' tool not found."
+
+    run jq -r .a.b < <(echo '{ "a": { "b" : "you found me" } }')
+    is "$output" "you found me" "sample invocation of 'jq'"
+}
+
+# vim: filetype=sh

test/system/005-info.bats

@@ -0,0 +1,54 @@
+#!/usr/bin/env bats
+
+load helpers
+
+@test "podman info - basic test" {
+    run_podman info
+
+    expected_keys="
+BuildahVersion: *[0-9.]\\\+
+Conmon:\\\s\\\+package:
+Distribution:
+OCIRuntime:\\\s\\\+package:
+os:
+rootless:
+insecure registries:
+store:
+GraphDriverName:
+GraphRoot:
+GraphStatus:
+ImageStore:\\\s\\\+number: 1
+RunRoot:
+"
+    while read expect; do
+        is "$output" ".*$expect" "output includes '$expect'"
+    done < <(parse_table "$expected_keys")
+}
+
+@test "podman info - json" {
+    run_podman info --format=json
+
+    expr_nvr="[a-z0-9-]\\\+-[a-z0-9.]\\\+-[a-z0-9]\\\+\."
+    expr_path="/[a-z0-9\\\/.]\\\+\\\$"
+
+    tests="
+host.BuildahVersion       | [0-9.]
+host.Conmon.package       | $expr_nvr
+host.Conmon.path          | $expr_path
+host.OCIRuntime.package   | $expr_nvr
+host.OCIRuntime.path      | $expr_path
+store.ConfigFile          | $expr_path
+store.GraphDriverName     | [a-z0-9]\\\+\\\$
+store.GraphRoot           | $expr_path
+store.ImageStore.number   | 1
+"
+
+    parse_table "$tests" | while read field expect; do
+        actual=$(echo "$output" | jq -r ".$field")
+        dprint "# actual=<$actual> expect=<$expect>"
+        is "$actual" "$expect" "jq .$field"
+    done
+
+}
+
+# vim: filetype=sh

test/system/010-images.bats

@@ -0,0 +1,46 @@
+#!/usr/bin/env bats
+
+load helpers
+
+@test "podman images - basic output" {
+    run_podman images -a
+
+    is "${lines[0]}" "REPOSITORY *TAG *IMAGE ID *CREATED *SIZE" "header line"
+    is "${lines[1]}" "$PODMAN_TEST_IMAGE_REGISTRY/$PODMAN_TEST_IMAGE_USER/$PODMAN_TEST_IMAGE_NAME *$PODMAN_TEST_IMAGE_TAG *[0-9a-f]\+" "podman images output"
+}
+
+@test "podman images - custom formats" {
+    tests="
+--format {{.ID}}                  |        [0-9a-f]\\\{12\\\}
+--format {{.ID}} --no-trunc       | sha256:[0-9a-f]\\\{64\\\}
+--format {{.Repository}}:{{.Tag}} | $PODMAN_TEST_IMAGE_FQN
+"
+
+    parse_table "$tests" | while read fmt expect; do
+        run_podman images $fmt
+        is "$output" "$expect\$" "podman images $fmt"
+    done
+
+}
+
+
+@test "podman images - json" {
+    tests="
+names[0]   | $PODMAN_TEST_IMAGE_FQN
+id         |        [0-9a-f]\\\{64\\\}
+digest     | sha256:[0-9a-f]\\\{64\\\}
+created    | [0-9-]\\\+T[0-9:]\\\+\\\.[0-9]\\\+Z
+size       | [0-9]\\\+
+"
+
+    run_podman images -a --format json
+
+    parse_table "$tests" | while read field expect; do
+        actual=$(echo "$output" | jq -r ".[0].$field")
+        dprint "# actual=<$actual> expect=<$expect}>"
+        is "$actual" "$expect" "jq .$field"
+    done
+
+}
+
+# vim: filetype=sh

test/system/015-help.bats

@@ -0,0 +1,76 @@
+#!/usr/bin/env bats
+#
+# Tests based on 'podman help'
+#
+# Find all commands listed by 'podman --help'. Run each one, make sure it
+# provides its own --help output. If the usage message ends in '[command]',
+# treat it as a subcommand, and recurse into its own list of sub-subcommands.
+#
+# Any usage message that ends in '[flags]' is interpreted as a command
+# that takes no further arguments; we confirm by running with 'invalid-arg'
+# and confirming that it exits with error status and message.
+#
+load helpers
+
+# run 'podman help', parse the output looking for 'Available Commands';
+# return that list.
+function podman_commands() {
+    dprint "$@"
+    run_podman help "$@" |\
+        awk '/^Available Commands:/{ok=1;next}/^Flags:/{ok=0}ok { print $1 }' |\
+        grep .
+    "$output"
+}
+
+
+function check_help() {
+    local count=0
+    local subcommands_found=0
+
+    for cmd in $(podman_commands "$@"); do
+        dprint "podman $@ $cmd --help"
+        run_podman "$@" $cmd --help
+
+        # The line immediately after 'Usage:' gives us a 1-line synopsis
+        usage=$(echo "$output" | grep -A1 '^Usage:' | tail -1)
+        [ -n "$usage" ] || die "podman $cmd: no Usage message found"
+
+        # If usage ends in '[command]', recurse into subcommands
+        if expr "$usage" : '.*\[command\]$' >/dev/null; then
+            subcommands_found=$(expr $subcommands_found + 1)
+            check_help "$@" $cmd
+            continue
+        fi
+
+        # If usage ends in '[flag]', command takes no more arguments.
+        # Confirm that by running with 'invalid-arg' and expecting failure.
+        if expr "$usage" : '.*\[flags\]$' >/dev/null; then
+            if [ "$cmd" != "help" ]; then
+                run_podman 125 "$@" $cmd invalid-arg
+                is "$output" "Error: .* takes no arguments" \
+                   "'podman $@ $cmd' with extra (invalid) arguments"
+            fi
+        fi
+
+        count=$(expr $count + 1)
+    done
+
+    # This can happen if the output of --help changes, such as between
+    # the old command parser and cobra.
+    [ $count -gt 0 ] || \
+        die "Internal error: no commands found in 'podman help $@' list"
+
+    # At least the top level must have some subcommands
+    if [ -z "$*" -a $subcommands_found -eq 0 ]; then
+        die "Internal error: did not find any podman subcommands"
+    fi
+}
+
+
+@test "podman help - basic tests" {
+    # Called with no args -- start with 'podman --help'. check_help() will
+    # recurse for any subcommands.
+    check_help
+}
+
+# vim: filetype=sh

test/system/030-run.bats

@@ -0,0 +1,34 @@
+#!/usr/bin/env bats
+
+load helpers
+
+@test "podman run - basic tests" {
+    rand=$(random_string 30)
+    tests="
+true              |   0 |
+false             |   1 |
+sh -c 'exit 32'   |  32 |
+echo $rand        |   0 | $rand
+/no/such/command  | 127 | Error: container create failed:.*exec:.* no such file or dir
+/etc              | 126 | Error: container create failed:.*exec:.* permission denied
+"
+
+    while read cmd expected_rc expected_output; do
+        if [ "$expected_output" = "''" ]; then expected_output=""; fi
+
+        # THIS IS TRICKY: this is what lets us handle a quoted command.
+        # Without this incantation (and the "$@" below), the cmd string
+        # gets passed on as individual tokens: eg "sh" "-c" "'exit" "32'"
+        # (note unmatched opening and closing single-quotes in the last 2).
+        # That results in a bizarre and hard-to-understand failure
+        # in the BATS 'run' invocation.
+        # This should really be done inside parse_table; I can't find
+        # a way to do so.
+        eval set "$cmd"
+
+        run_podman $expected_rc run $IMAGE "$@"
+        is "$output" "$expected_output" "podman run $cmd - output"
+    done < <(parse_table "$tests")
+}
+
+# vim: filetype=sh