System tests: add test tags

BATS 1.8.0 introduces tags: metadata that can be applied to a single test or one entire file, then used for filtering in a test run. Issue #19299 introduces the possibility of using OpenQA for podman reverse dependency testing: continuous CI on all packages that can affect podman, so we don't go two months with no bodhi builds then get caught by surprise when systemd or kernel or crun change in ways that break us. This PR introduces one bats tag, "openqa". The intention is for OpenQA tests to install the podman-tests package and run: bats --filter-tags openqa /usr/share/podman/test/system Goal is to keep the test list short and sweet: we do not need to test command-line option parsing. We *DO* need to test interactions with systemd, kernel, nethack, and other critical components. Signed-off-by: Ed Santiago <santiago@redhat.com>
containers · Jul 26, 2023 · 83a9f29 · 83a9f29
1 parent eac1e94
commit 83a9f29
Show file tree

Hide file tree

Showing 18 changed files with 41 additions and 0 deletions.
diff --git a/hack/bats b/hack/bats
@@ -83,6 +83,7 @@ for i;do
         --rootless) TEST_ROOT= ;;
         --remote)   REMOTE=remote ;;
         --ts|-T)    bats_opts+=("-T") ;;
+        --tag=*)    bats_filter=("--filter-tags" "$value") ;;
         */*.bats)   TESTS=$i ;;
         *)
             if [[ $i =~ : ]]; then

diff --git a/test/system/001-basic.bats b/test/system/001-basic.bats
@@ -13,6 +13,7 @@ function setup() {
 
 #### DO NOT ADD ANY TESTS HERE! ADD NEW TESTS AT BOTTOM!
 
+# bats test_tags=openqa
 @test "podman version emits reasonable output" {
     run_podman version
 
@@ -38,6 +39,7 @@ function setup() {
     is "$output" ".*The --config flag is ignored by Podman. Exists for Docker compatibility\+"		  "verify warning for --config option"
 }
 
+# bats test_tags=openqa
 @test "podman info" {
     # These will be displayed on the test output stream, offering an
     # at-a-glance overview of important system configuration details
@@ -71,6 +73,7 @@ function setup() {
        "--context=swarm should fail"
 }
 
+# bats test_tags=openqa
 @test "podman can pull an image" {
     run_podman rmi -a -f
 

diff --git a/test/system/030-run.bats b/test/system/030-run.bats
@@ -3,6 +3,7 @@
 load helpers
 load helpers.network
 
+# bats test_tags=openqa
 @test "podman run - basic tests" {
     rand=$(random_string 30)
 
@@ -296,6 +297,7 @@ echo $rand        |   0 | $rand
 }
 
 # #6829 : add username to /etc/passwd inside container if --userns=keep-id
+# bats test_tags=openqa
 @test "podman run : add username to /etc/passwd if --userns=keep-id" {
     skip_if_not_rootless "--userns=keep-id only works in rootless mode"
     # Default: always run as root
@@ -695,6 +697,7 @@ json-file | f
 # https://github.com/containers/podman/issues/9096
 # podman exec may truncate stdout/stderr; actually a bug in conmon:
 # https://github.com/containers/conmon/issues/236
+# bats test_tags=openqa
 @test "podman run - does not truncate or hang with big output" {
     # Size, in bytes, to dd and to expect in return
     char_count=700000

diff --git a/test/system/032-sig-proxy.bats b/test/system/032-sig-proxy.bats
@@ -5,6 +5,7 @@ load helpers.sig-proxy
 
 # Each of the tests below does some setup, then invokes the helper from helpers.sig-proxy.bash.
 
+# bats test_tags=openqa
 @test "podman sigproxy test: run" {
     # We're forced to use $PODMAN because run_podman cannot be backgrounded
     $PODMAN run -i --name c_run $IMAGE sh -c "$SLEEPLOOP" &

diff --git a/test/system/035-logs.bats b/test/system/035-logs.bats
@@ -337,6 +337,7 @@ timeout: sending signal TERM to command.*" "logs --since -f on running container
     _log_test_follow_since k8s-file
 }
 
+# bats test_tags=openqa
 @test "podman logs - --since --follow journald" {
     # We can't use journald on RHEL as rootless: rhbz#1895105
     skip_if_journald_unavailable
@@ -379,6 +380,7 @@ $content--2.*" "logs --until -f on running container works"
     _log_test_follow_until k8s-file
 }
 
+# bats test_tags=openqa
 @test "podman logs - --until --follow journald" {
     # We can't use journald on RHEL as rootless: rhbz#1895105
     skip_if_journald_unavailable

diff --git a/test/system/070-build.bats b/test/system/070-build.bats
@@ -6,6 +6,7 @@
 
 load helpers
 
+# bats test_tags=openqa
 @test "podman build - basic test" {
     rand_filename=$(random_string 20)
     rand_content=$(random_string 50)
@@ -267,6 +268,7 @@ EOF
 }
 
 
+# bats test_tags=openqa
 @test "podman build - workdir, cmd, env, label" {
     tmpdir=$PODMAN_TMPDIR/build-test
     mkdir -p $tmpdir

diff --git a/test/system/075-exec.bats b/test/system/075-exec.bats
@@ -5,6 +5,7 @@
 
 load helpers
 
+# bats test_tags=openqa
 @test "podman exec - basic test" {
     rand_filename=$(random_string 20)
     rand_content=$(random_string 50)
@@ -43,6 +44,7 @@ load helpers
     run_podman rm $cid
 }
 
+# bats test_tags=openqa
 @test "podman exec - leak check" {
     skip_if_remote "test is meaningless over remote"
 

diff --git a/test/system/080-pause.bats b/test/system/080-pause.bats
@@ -5,6 +5,7 @@
 
 load helpers
 
+# bats test_tags=openqa
 @test "podman pause/unpause" {
     if is_rootless && ! is_cgroupsv2; then
         skip "'podman pause' (rootless) only works with cgroups v2"
@@ -58,6 +59,7 @@ load helpers
     run_podman 125 unpause $cname
 }
 
+# bats test_tags=openqa
 @test "podman unpause --all" {
     if is_rootless && ! is_cgroupsv2; then
         skip "'podman pause' (rootless) only works with cgroups v2"

diff --git a/test/system/090-events.bats b/test/system/090-events.bats
@@ -5,6 +5,7 @@
 
 load helpers
 
+# bats test_tags=openqa
 @test "events with a filter by label" {
     cname=test-$(random_string 30 | tr A-Z a-z)
     labelname=$(random_string 10)
@@ -141,6 +142,7 @@ function _events_disjunctive_filters() {
     _events_disjunctive_filters ""
 }
 
+# bats test_tags=openqa
 @test "events with events_logfile_path in containers.conf" {
     skip_if_remote "remote does not support --events-backend"
     events_file=$PODMAN_TMPDIR/events.log
@@ -162,6 +164,7 @@ function _populate_events_file() {
     done
 }
 
+# bats test_tags=openqa
 @test "events log-file rotation" {
     skip_if_remote "setting CONTAINERS_CONF_OVERRIDE logger options does not affect remote client"
 

diff --git a/test/system/130-kill.bats b/test/system/130-kill.bats
@@ -5,6 +5,7 @@
 
 load helpers
 
+# bats test_tags=openqa
 @test "podman kill - test signal handling in containers" {
 
     # Prepare for 'logs -f'

diff --git a/test/system/160-volumes.bats b/test/system/160-volumes.bats
@@ -395,6 +395,7 @@ EOF
     is "$output"  "" "no more volumes to prune"
 }
 
+# bats test_tags=openqa
 @test "podman volume type=bind" {
     myvoldir=${PODMAN_TMPDIR}/volume_$(random_string)
     mkdir $myvoldir

diff --git a/test/system/170-run-userns.bats b/test/system/170-run-userns.bats
@@ -3,6 +3,8 @@
 #
 # Tests for podman build
 #
+# bats file_tags=openqa
+#
 
 load helpers
 

diff --git a/test/system/180-blkio.bats b/test/system/180-blkio.bats
@@ -2,6 +2,8 @@
 #
 # podman blkio-related tests
 #
+# bats file_tags=openqa
+#
 
 load helpers
 

diff --git a/test/system/190-run-ipcns.bats b/test/system/190-run-ipcns.bats
@@ -3,6 +3,8 @@
 #
 # Tests for podman build
 #
+# bats file_tags=openqa
+#
 
 load helpers
 

diff --git a/test/system/195-run-namesapces.bats → test/system/195-run-namespaces.bats b/test/system/195-run-namesapces.bats → test/system/195-run-namespaces.bats
@@ -5,6 +5,7 @@
 
 load helpers
 
+# bats test_tags=openqa
 @test "podman test all namespaces" {
     # format is nsname | option name
     tests="

diff --git a/test/system/260-sdnotify.bats b/test/system/260-sdnotify.bats
@@ -103,6 +103,7 @@ function _assert_mainpid_is_conmon() {
     _stop_socat
 }
 
+# bats test_tags=openqa
 @test "sdnotify : conmon" {
     export NOTIFY_SOCKET=$PODMAN_TMPDIR/conmon.sock
     _start_socat
@@ -141,6 +142,7 @@ READY=1" "sdnotify sent MAINPID and READY"
 
 # These tests can fail in dev. environment because of SELinux.
 # quick fix: chcon -t container_runtime_exec_t ./bin/podman
+# bats test_tags=openqa
 @test "sdnotify : container" {
     _prefetch $SYSTEMD_IMAGE
 
@@ -452,6 +454,7 @@ spec:
 " > $fname
 }
 
+# bats test_tags=openqa
 @test "podman kube play - exit-code propagation" {
     fname=$PODMAN_TMPDIR/$(random_string).yaml
 

diff --git a/test/system/410-selinux.bats b/test/system/410-selinux.bats
@@ -34,26 +34,32 @@ function check_label() {
 }
 
 
+# bats test_tags=openqa
 @test "podman selinux: confined container" {
     check_label "" "container_t"
 }
 
+# FIXME #19376 - container-selinux broken -- bats test_tags=openqa
 @test "podman selinux: container with label=disable" {
     check_label "--security-opt label=disable" "spc_t"
 }
 
+# FIXME #19376 - container-selinux broken -- bats test_tags=openqa
 @test "podman selinux: privileged container" {
     check_label "--privileged --userns=host" "spc_t"
 }
 
+# bats test_tags=openqa
 @test "podman selinux: init container" {
     check_label "--systemd=always" "container_init_t"
 }
 
+# bats test_tags=openqa
 @test "podman selinux: init container with --security-opt type" {
     check_label "--systemd=always --security-opt=label=type:spc_t" "spc_t"
 }
 
+# bats test_tags=openqa
 @test "podman selinux: init container with --security-opt level&type" {
     check_label "--systemd=always --security-opt=label=level:s0:c1,c2 --security-opt=label=type:spc_t" "spc_t" "s0:c1,c2"
 }
@@ -62,6 +68,7 @@ function check_label() {
     check_label "--systemd=always --security-opt=label=level:s0:c1,c2" "container_init_t"  "s0:c1,c2"
 }
 
+# FIXME #19376 - container-selinux broken -- bats test_tags=openqa
 @test "podman selinux: pid=host" {
     # FIXME this test fails when run rootless with runc:
     #   Error: container_linux.go:367: starting container process caused: process_linux.go:495: container init caused: readonly path /proc/asound: operation not permitted: OCI permission denied
@@ -153,6 +160,7 @@ function check_label() {
 }
 
 # pr #7902 - containers in pods should all run under same context
+# bats test_tags=openqa
 @test "podman selinux: containers in pods share full context" {
     skip_if_no_selinux
 
@@ -226,6 +234,7 @@ function check_label() {
     is "$output" "Error.*: $expect" "podman emits useful diagnostic on failure"
 }
 
+# bats test_tags=openqa
 @test "podman selinux: check relabel" {
     skip_if_no_selinux
 

diff --git a/test/system/500-networking.bats b/test/system/500-networking.bats
@@ -449,6 +449,7 @@ load helpers.network
 }
 
 # Test for https://github.com/containers/podman/issues/10052
+# bats test_tags=openqa
 @test "podman network connect/disconnect with port forwarding" {
     random_1=$(random_string 30)
     HOST_PORT=$(random_free_port)