Merge remote-tracking branch 'remotes/huth-gitlab/tags/pull-request-2…

…021-01-27' into staging

* Patches to speed up and improve the gitlab-CI
* Documentation for the decorators in the "acceptance" tests
* One small rework of a libqtest function

# gpg: Signature made Wed 27 Jan 2021 06:22:11 GMT
# gpg:                using RSA key 27B88847EEE0250118F3EAB92ED9D774FE702DB5
# gpg:                issuer "thuth@redhat.com"
# gpg: Good signature from "Thomas Huth <th.huth@gmx.de>" [full]
# gpg:                 aka "Thomas Huth <thuth@redhat.com>" [full]
# gpg:                 aka "Thomas Huth <huth@tuxfamily.org>" [full]
# gpg:                 aka "Thomas Huth <th.huth@posteo.de>" [unknown]
# Primary key fingerprint: 27B8 8847 EEE0 2501 18F3  EAB9 2ED9 D774 FE70 2DB5

* remotes/huth-gitlab/tags/pull-request-2021-01-27:
  libqtest: Rework qtest_rsp()
  docs/devel: Explain how acceptance tests can be skipped
  gitlab-ci.yml: Avoid recompiling the sources in the test jobs
  gitlab-ci.yml: Exclude some redundant targets in build-without-default-features
  meson: Do not build optional libraries by default
  configure: Only check for audio drivers if system-mode is selected
  gitlab-ci.yml: Avoid some submodules to speed up the CI a little bit
  gitlab-ci: Test building linux-user targets on CentOS 7
  tests/docker: Install static libc package in CentOS 7

Signed-off-by: Peter Maydell <peter.maydell@linaro.org>

.gitlab-ci.yml

@@ -38,9 +38,12 @@ include:
   stage: test
   image: $CI_REGISTRY_IMAGE/qemu/$IMAGE:latest
   script:
+    - scripts/git-submodule.sh update
+        $(grep GIT_SUBMODULES build/config-host.mak | sed 's/GIT_SUBMODULES=//')
     - cd build
     - find . -type f -exec touch {} +
-    - make $MAKE_CHECK_ARGS
+    # Avoid recompiling by hiding ninja with NINJA=":"
+    - make NINJA=":" $MAKE_CHECK_ARGS
 
 .acceptance_template: &acceptance_definition
   cache:
@@ -109,6 +112,7 @@ build-system-ubuntu:
   <<: *native_build_job_definition
   variables:
     IMAGE: ubuntu2004
+    CONFIGURE_ARGS: --enable-fdt=system --enable-slirp=system
     TARGETS: aarch64-softmmu alpha-softmmu cris-softmmu hppa-softmmu
       moxie-softmmu microblazeel-softmmu mips64el-softmmu
     MAKE_CHECK_ARGS: check-build
@@ -140,6 +144,7 @@ build-system-debian:
   <<: *native_build_job_definition
   variables:
     IMAGE: debian-amd64
+    CONFIGURE_ARGS: --enable-fdt=system
     TARGETS: arm-softmmu avr-softmmu i386-softmmu mipsel-softmmu
       riscv64-softmmu sh4eb-softmmu sparc-softmmu xtensaeb-softmmu
     MAKE_CHECK_ARGS: check-build
@@ -185,6 +190,7 @@ build-system-fedora:
   variables:
     IMAGE: fedora
     CONFIGURE_ARGS: --disable-gcrypt --enable-nettle --enable-docs
+             --enable-fdt=system --enable-slirp=system --enable-capstone=system
     TARGETS: tricore-softmmu microblaze-softmmu mips-softmmu
       xtensa-softmmu m68k-softmmu riscv32-softmmu ppc-softmmu sparc64-softmmu
     MAKE_CHECK_ARGS: check-build
@@ -216,7 +222,7 @@ build-system-centos:
   <<: *native_build_job_definition
   variables:
     IMAGE: centos8
-    CONFIGURE_ARGS: --disable-nettle --enable-gcrypt
+    CONFIGURE_ARGS: --disable-nettle --enable-gcrypt --enable-fdt=system
     TARGETS: ppc64-softmmu or1k-softmmu s390x-softmmu
       x86_64-softmmu rx-softmmu sh4-softmmu nios2-softmmu
     MAKE_CHECK_ARGS: check-build
@@ -248,6 +254,7 @@ build-system-opensuse:
   <<: *native_build_job_definition
   variables:
     IMAGE: opensuse-leap
+    CONFIGURE_ARGS: --enable-fdt=system
     TARGETS: s390x-softmmu x86_64-softmmu aarch64-softmmu
     MAKE_CHECK_ARGS: check-build
   artifacts:
@@ -411,6 +418,13 @@ build-user-plugins:
     MAKE_CHECK_ARGS: check-tcg
   timeout: 1h 30m
 
+build-user-centos7:
+  <<: *native_build_job_definition
+  variables:
+    IMAGE: centos7
+    CONFIGURE_ARGS: --disable-system --disable-tools --disable-docs
+    MAKE_CHECK_ARGS: check-tcg
+
 build-some-softmmu-plugins:
   <<: *native_build_job_definition
   variables:
@@ -606,6 +620,7 @@ build-without-default-features:
   variables:
     IMAGE: debian-amd64
     CONFIGURE_ARGS: --without-default-features --disable-user
+        --target-list-exclude=arm-softmmu,i386-softmmu,mipsel-softmmu,mips64-softmmu,ppc-softmmu
     MAKE_CHECK_ARGS: check-unit
 
 check-patch:

configure

@@ -2319,6 +2319,12 @@ if test -z "$want_tools"; then
     fi
 fi
 
+##########################################
+# Disable features only meaningful for system-mode emulation
+if test "$softmmu" = "no"; then
+    audio_drv_list=""
+fi
+
 ##########################################
 # Some versions of Mac OS X incorrectly define SIZE_MAX
 cat > $TMPC << EOF

docs/devel/testing.rst

@@ -871,6 +871,68 @@ qemu_bin
 
 The exact QEMU binary to be used on QEMUMachine.
 
+Skipping tests
+--------------
+The Avocado framework provides Python decorators which allow for easily skip
+tests running under certain conditions. For example, on the lack of a binary
+on the test system or when the running environment is a CI system. For further
+information about those decorators, please refer to::
+
+  https://avocado-framework.readthedocs.io/en/latest/guides/writer/chapters/writing.html#skipping-tests
+
+While the conditions for skipping tests are often specifics of each one, there
+are recurring scenarios identified by the QEMU developers and the use of
+environment variables became a kind of standard way to enable/disable tests.
+
+Here is a list of the most used variables:
+
+AVOCADO_ALLOW_LARGE_STORAGE
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Tests which are going to fetch or produce assets considered *large* are not
+going to run unless that `AVOCADO_ALLOW_LARGE_STORAGE=1` is exported on
+the environment.
+
+The definition of *large* is a bit arbitrary here, but it usually means an
+asset which occupies at least 1GB of size on disk when uncompressed.
+
+AVOCADO_ALLOW_UNTRUSTED_CODE
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+There are tests which will boot a kernel image or firmware that can be
+considered not safe to run on the developer's workstation, thus they are
+skipped by default. The definition of *not safe* is also arbitrary but
+usually it means a blob which either its source or build process aren't
+public available.
+
+You should export `AVOCADO_ALLOW_UNTRUSTED_CODE=1` on the environment in
+order to allow tests which make use of those kind of assets.
+
+AVOCADO_TIMEOUT_EXPECTED
+~~~~~~~~~~~~~~~~~~~~~~~~
+The Avocado framework has a timeout mechanism which interrupts tests to avoid the
+test suite of getting stuck. The timeout value can be set via test parameter or
+property defined in the test class, for further details::
+
+  https://avocado-framework.readthedocs.io/en/latest/guides/writer/chapters/writing.html#setting-a-test-timeout
+
+Even though the timeout can be set by the test developer, there are some tests
+that may not have a well-defined limit of time to finish under certain
+conditions. For example, tests that take longer to execute when QEMU is
+compiled with debug flags. Therefore, the `AVOCADO_TIMEOUT_EXPECTED` variable
+has been used to determine whether those tests should run or not.
+
+GITLAB_CI
+~~~~~~~~~
+A number of tests are flagged to not run on the GitLab CI. Usually because
+they proved to the flaky or there are constraints on the CI environment which
+would make them fail. If you encounter a similar situation then use that
+variable as shown on the code snippet below to skip the test:
+
+.. code::
+
+  @skipIf(os.getenv('GITLAB_CI'), 'Running on GitLab')
+  def test(self):
+      do_something()
+
 Uninstalling Avocado
 --------------------
 

meson.build

@@ -1460,6 +1460,7 @@ if capstone_opt == 'internal'
   ]
 
   libcapstone = static_library('capstone',
+                               build_by_default: false,
                                sources: capstone_files,
                                c_args: capstone_cargs,
                                include_directories: 'capstone/include')
@@ -1537,6 +1538,7 @@ if have_system
 
     slirp_inc = include_directories('slirp', 'slirp/src')
     libslirp = static_library('slirp',
+                              build_by_default: false,
                               sources: slirp_files,
                               c_args: slirp_cargs,
                               include_directories: slirp_inc)
@@ -1582,6 +1584,7 @@ if have_system
 
     fdt_inc = include_directories('dtc/libfdt')
     libfdt = static_library('fdt',
+                            build_by_default: false,
                             sources: fdt_files,
                             include_directories: fdt_inc)
     fdt = declare_dependency(link_with: libfdt,

tests/docker/dockerfiles/centos7.docker

@@ -15,6 +15,7 @@ ENV PACKAGES \
     gettext \
     git \
     glib2-devel \
+    glibc-static \
     gnutls-devel \
     libaio-devel \
     libepoxy-devel \

tests/docker/dockerfiles/centos8.docker

@@ -15,6 +15,7 @@ ENV PACKAGES \
     glib2-devel \
     libaio-devel \
     libepoxy-devel \
+    libfdt-devel \
     libgcrypt-devel \
     lzo-devel \
     make \

tests/docker/dockerfiles/debian-amd64.docker

@@ -21,6 +21,7 @@ RUN apt update && \
         libbz2-dev \
         liblzo2-dev \
         libgcrypt20-dev \
+        libfdt-dev \
         librdmacm-dev \
         libsasl2-dev \
         libsnappy-dev \

tests/docker/dockerfiles/fedora.docker

@@ -6,6 +6,7 @@ ENV PACKAGES \
     brlapi-devel \
     bzip2 \
     bzip2-devel \
+    capstone-devel \
     ccache \
     clang \
     cyrus-sasl-devel \
@@ -37,6 +38,7 @@ ENV PACKAGES \
     libpng-devel \
     librbd-devel \
     libseccomp-devel \
+    libslirp-devel \
     libssh-devel \
     libubsan \
     libudev-devel \
@@ -46,6 +48,7 @@ ENV PACKAGES \
     llvm \
     lzo-devel \
     make \
+    meson \
     mingw32-bzip2 \
     mingw32-curl \
     mingw32-glib2 \

tests/docker/dockerfiles/ubuntu2004.docker

@@ -37,6 +37,7 @@ ENV PACKAGES flex bison \
     libsasl2-dev \
     libsdl2-dev \
     libseccomp-dev \
+    libslirp-dev \
     libsnappy-dev \
     libspice-protocol-dev \
     libspice-server-dev \