Merge tag 'pull-request-2023-11-16' of https://gitlab.com/thuth/qemu …

…into staging

* Fix the avocado tests for running with Python 3.12
* Add some asset hashes to silence warnings
* Fix the broken reverse_debugging test

# -----BEGIN PGP SIGNATURE-----
#
# iQJFBAABCAAvFiEEJ7iIR+7gJQEY8+q5LtnXdP5wLbUFAmVWWRARHHRodXRoQHJl
# ZGhhdC5jb20ACgkQLtnXdP5wLbWOnQ/8DdnPGlXrOl+4QzRaJQovmmS04OZuiWtB
# TpOd/cWQMsKKX7YGq9uAuLBa7zYXyHZLmOLcSYYKYzcFnxX155TgtXDqZXJb7k8D
# 9rGtF8vgGokT3VmsWW6w5LNa1Pg5FPd56+l8Kus7iqpHbxPzMybP/HFGcuCdeMW9
# kFaLCvtP7LAPM9k8z6Nw9vbQ5aUx2J9fwSgWhoMYmVO72A7HUZ0KA+lti/3xvhKy
# zcYJ4PxFaSK4SQwG8yfhmwqCw3QjeH+Se8gxvqR6nmKW9wCA3FdmUFaQPQQF6bq+
# AEUPBJpa+eZA38tA9rts6VSVGgrg5IA9rPYFc8ZIYn6dMgkAqR7snvcphyUAijJ9
# oqS72BJkw6KZ+52QyZ4O2zG4lkgi/uHp0qhmAr/FHDLkmetTjyQ4vA5dm76V5B2G
# EAPzNZct0RluD0hk2qIKXRC4FN4HFaEAL4EpDjCuymTArsEhSf9I8IEMu9V4CVHs
# Jp2c/0EZNCcTeqc7xr+lO+JFMd1kbB2N9yApFrbQpaIsZQkSF/oNtFZ8h8n/DhNJ
# Ojp9hSUkb37eBZuRA5wsDAo9WI1ADV1n7Ku7ao9aJ84iozcAvOjk2T+NXkeKJEdW
# br8RCqpm0yiB9W7X9WUa5uQd69pdKX4ILW1ekdrG02CuxbIE42Jx5WYdzUrXDUIz
# hzD/QPF3TsI=
# =IcSd
# -----END PGP SIGNATURE-----
# gpg: Signature made Thu 16 Nov 2023 13:01:52 EST
# gpg:                using RSA key 27B88847EEE0250118F3EAB92ED9D774FE702DB5
# gpg:                issuer "thuth@redhat.com"
# gpg: Good signature from "Thomas Huth <th.huth@gmx.de>" [full]
# gpg:                 aka "Thomas Huth <thuth@redhat.com>" [full]
# gpg:                 aka "Thomas Huth <huth@tuxfamily.org>" [full]
# gpg:                 aka "Thomas Huth <th.huth@posteo.de>" [unknown]
# Primary key fingerprint: 27B8 8847 EEE0 2501 18F3  EAB9 2ED9 D774 FE70 2DB5

* tag 'pull-request-2023-11-16' of https://gitlab.com/thuth/qemu:
  tests/avocado: Enable reverse_debugging.py tests in gitlab CI
  tests/avocado: reverse_debugging drain console to prevent hang
  tests/avocado/mem-addr-space-check: Replace assertEquals() for Python 3.12
  tests/avocado/replay_kernel: Mark the test_x86_64_pc as flaky
  tests/avocado: Make fetch_asset() unconditionally require a crypto hash
  tests/avocado/multiprocess: Add asset hashes to silence warnings
  tests/avocado/intel_iommu: Add asset hashes to avoid warnings
  tests/avocado/virtio-gpu: Fix test_vhost_user_vga_virgl for edid support
  tests/avocado: Replace assertRegexpMatches() for Python 3.12 compatibility
  tests/avocado: Replace assertEquals() for Python 3.12 compatibility

Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>

docs/devel/testing.rst

@@ -1016,7 +1016,7 @@ class.  Here's a simple usage example:
           self.vm.launch()
           res = self.vm.cmd('human-monitor-command',
                             command_line='info version')
-          self.assertRegexpMatches(res, r'^(\d+\.\d+\.\d)')
+          self.assertRegex(res, r'^(\d+\.\d+\.\d)')
 
 To execute your test, run:
 
@@ -1077,7 +1077,7 @@ and hypothetical example follows:
               'human-monitor-command',
               command_line='info version')
 
-          self.assertEquals(first_res, second_res, third_res)
+          self.assertEqual(first_res, second_res, third_res)
 
 At test "tear down", ``avocado_qemu.Test`` handles all the QEMUMachines
 shutdown.

tests/avocado/avocado_qemu/__init__.py

@@ -254,7 +254,7 @@ def setUp(self, bin_prefix):
             self.cancel("No QEMU binary defined or found in the build tree")
 
     def fetch_asset(self, name,
-                    asset_hash=None, algorithm=None,
+                    asset_hash, algorithm=None,
                     locations=None, expire=None,
                     find_only=False, cancel_on_missing=True):
         return super().fetch_asset(name,

tests/avocado/cpu_queries.py

@@ -32,4 +32,4 @@ def test(self):
             model = {'name': c['name']}
             e = self.vm.cmd('query-cpu-model-expansion', model=model,
                             type='full')
-            self.assertEquals(e['model']['name'], c['name'])
+            self.assertEqual(e['model']['name'], c['name'])

tests/avocado/empty_cpu_model.py

@@ -15,5 +15,5 @@ def test(self):
         self.vm.set_qmp_monitor(enabled=False)
         self.vm.launch()
         self.vm.wait()
-        self.assertEquals(self.vm.exitcode(), 1, "QEMU exit code should be 1")
+        self.assertEqual(self.vm.exitcode(), 1, "QEMU exit code should be 1")
         self.assertRegex(self.vm.get_log(), r'-cpu option cannot be empty')

tests/avocado/intel_iommu.py

@@ -54,9 +54,11 @@ def common_vm_setup(self, custom_kernel=None):
             return
 
         kernel_url = self.distro.pxeboot_url + 'vmlinuz'
+        kernel_hash = '5b6f6876e1b5bda314f93893271da0d5777b1f3c'
         initrd_url = self.distro.pxeboot_url + 'initrd.img'
-        self.kernel_path = self.fetch_asset(kernel_url)
-        self.initrd_path = self.fetch_asset(initrd_url)
+        initrd_hash = 'dd0340a1b39bd28f88532babd4581c67649ec5b1'
+        self.kernel_path = self.fetch_asset(kernel_url, asset_hash=kernel_hash)
+        self.initrd_path = self.fetch_asset(initrd_url, asset_hash=initrd_hash)
 
     def run_and_check(self):
         if self.kernel_path:

tests/avocado/mem-addr-space-check.py

@@ -49,7 +49,7 @@ def test_phybits_low_pse36(self):
         self.vm.set_qmp_monitor(enabled=False)
         self.vm.launch()
         self.vm.wait()
-        self.assertEquals(self.vm.exitcode(), 1, "QEMU exit code should be 1")
+        self.assertEqual(self.vm.exitcode(), 1, "QEMU exit code should be 1")
         self.assertRegex(self.vm.get_log(), r'phys-bits too low')
 
     def test_phybits_low_pae(self):
@@ -69,7 +69,7 @@ def test_phybits_low_pae(self):
         self.vm.set_qmp_monitor(enabled=False)
         self.vm.launch()
         self.vm.wait()
-        self.assertEquals(self.vm.exitcode(), 1, "QEMU exit code should be 1")
+        self.assertEqual(self.vm.exitcode(), 1, "QEMU exit code should be 1")
         self.assertRegex(self.vm.get_log(), r'phys-bits too low')
 
     def test_phybits_ok_pentium_pse36(self):
@@ -149,7 +149,7 @@ def test_phybits_low_nonpse36(self):
         self.vm.set_qmp_monitor(enabled=False)
         self.vm.launch()
         self.vm.wait()
-        self.assertEquals(self.vm.exitcode(), 1, "QEMU exit code should be 1")
+        self.assertEqual(self.vm.exitcode(), 1, "QEMU exit code should be 1")
         self.assertRegex(self.vm.get_log(), r'phys-bits too low')
 
     # now lets test some 64-bit CPU cases.
@@ -179,7 +179,7 @@ def test_phybits_low_tcg_q35_70_amd(self):
         self.vm.set_qmp_monitor(enabled=False)
         self.vm.launch()
         self.vm.wait()
-        self.assertEquals(self.vm.exitcode(), 1, "QEMU exit code should be 1")
+        self.assertEqual(self.vm.exitcode(), 1, "QEMU exit code should be 1")
         self.assertRegex(self.vm.get_log(), r'phys-bits too low')
 
     def test_phybits_low_tcg_q35_71_amd(self):
@@ -202,7 +202,7 @@ def test_phybits_low_tcg_q35_71_amd(self):
         self.vm.set_qmp_monitor(enabled=False)
         self.vm.launch()
         self.vm.wait()
-        self.assertEquals(self.vm.exitcode(), 1, "QEMU exit code should be 1")
+        self.assertEqual(self.vm.exitcode(), 1, "QEMU exit code should be 1")
         self.assertRegex(self.vm.get_log(), r'phys-bits too low')
 
     def test_phybits_ok_tcg_q35_70_amd(self):
@@ -288,7 +288,7 @@ def test_phybits_low_tcg_q35_71_amd_41bits(self):
         self.vm.set_qmp_monitor(enabled=False)
         self.vm.launch()
         self.vm.wait()
-        self.assertEquals(self.vm.exitcode(), 1, "QEMU exit code should be 1")
+        self.assertEqual(self.vm.exitcode(), 1, "QEMU exit code should be 1")
         self.assertRegex(self.vm.get_log(), r'phys-bits too low')
 
     def test_phybits_ok_tcg_q35_71_amd_41bits(self):
@@ -332,7 +332,7 @@ def test_phybits_low_tcg_q35_intel_cxl(self):
         self.vm.set_qmp_monitor(enabled=False)
         self.vm.launch()
         self.vm.wait()
-        self.assertEquals(self.vm.exitcode(), 1, "QEMU exit code should be 1")
+        self.assertEqual(self.vm.exitcode(), 1, "QEMU exit code should be 1")
         self.assertRegex(self.vm.get_log(), r'phys-bits too low')
 
     def test_phybits_ok_tcg_q35_intel_cxl(self):

tests/avocado/multiprocess.py

@@ -18,8 +18,8 @@ class Multiprocess(QemuSystemTest):
     """
     KERNEL_COMMON_COMMAND_LINE = 'printk.time=0 '
 
-    def do_test(self, kernel_url, initrd_url, kernel_command_line,
-                machine_type):
+    def do_test(self, kernel_url, kernel_hash, initrd_url, initrd_hash,
+                kernel_command_line, machine_type):
         """Main test method"""
         self.require_accelerator('kvm')
         self.require_multiprocess()
@@ -30,8 +30,8 @@ def do_test(self, kernel_url, initrd_url, kernel_command_line,
         os.set_inheritable(proxy_sock.fileno(), True)
         os.set_inheritable(remote_sock.fileno(), True)
 
-        kernel_path = self.fetch_asset(kernel_url)
-        initrd_path = self.fetch_asset(initrd_url)
+        kernel_path = self.fetch_asset(kernel_url, asset_hash=kernel_hash)
+        initrd_path = self.fetch_asset(initrd_url, asset_hash=initrd_hash)
 
         # Create remote process
         remote_vm = self.get_vm()
@@ -72,13 +72,16 @@ def test_multiprocess_x86_64(self):
         kernel_url = ('https://archives.fedoraproject.org/pub/archive/fedora'
                       '/linux/releases/31/Everything/x86_64/os/images'
                       '/pxeboot/vmlinuz')
+        kernel_hash = '5b6f6876e1b5bda314f93893271da0d5777b1f3c'
         initrd_url = ('https://archives.fedoraproject.org/pub/archive/fedora'
                       '/linux/releases/31/Everything/x86_64/os/images'
                       '/pxeboot/initrd.img')
+        initrd_hash = 'dd0340a1b39bd28f88532babd4581c67649ec5b1'
         kernel_command_line = (self.KERNEL_COMMON_COMMAND_LINE +
                                'console=ttyS0 rdinit=/bin/bash')
         machine_type = 'pc'
-        self.do_test(kernel_url, initrd_url, kernel_command_line, machine_type)
+        self.do_test(kernel_url, kernel_hash, initrd_url, initrd_hash,
+                     kernel_command_line, machine_type)
 
     def test_multiprocess_aarch64(self):
         """
@@ -87,10 +90,13 @@ def test_multiprocess_aarch64(self):
         kernel_url = ('https://archives.fedoraproject.org/pub/archive/fedora'
                       '/linux/releases/31/Everything/aarch64/os/images'
                       '/pxeboot/vmlinuz')
+        kernel_hash = '3505f2751e2833c681de78cee8dda1e49cabd2e8'
         initrd_url = ('https://archives.fedoraproject.org/pub/archive/fedora'
                       '/linux/releases/31/Everything/aarch64/os/images'
                       '/pxeboot/initrd.img')
+        initrd_hash = '519a1962daf17d67fc3a9c89d45affcb399607db'
         kernel_command_line = (self.KERNEL_COMMON_COMMAND_LINE +
                                'rdinit=/bin/bash console=ttyAMA0')
         machine_type = 'virt,gic-version=3'
-        self.do_test(kernel_url, initrd_url, kernel_command_line, machine_type)
+        self.do_test(kernel_url, kernel_hash, initrd_url, initrd_hash,
+                     kernel_command_line, machine_type)

tests/avocado/pc_cpu_hotplug_props.py

@@ -32,4 +32,4 @@ def test_no_die_id(self):
         self.vm.add_args('-smp', '1,sockets=2,cores=2,threads=2,maxcpus=8')
         self.vm.add_args('-device', 'qemu64-x86_64-cpu,socket-id=1,core-id=0,thread-id=0')
         self.vm.launch()
-        self.assertEquals(len(self.vm.cmd('query-cpus-fast')), 2)
+        self.assertEqual(len(self.vm.cmd('query-cpus-fast')), 2)

tests/avocado/replay_kernel.py

@@ -81,7 +81,8 @@ def run_rr(self, kernel_path, kernel_command_line, console_pattern,
         logger.info('replay overhead {:.2%}'.format(t2 / t1 - 1))
 
 class ReplayKernelNormal(ReplayKernelBase):
-    @skipIf(os.getenv('GITLAB_CI'), 'Running on GitLab')
+
+    @skipUnless(os.getenv('QEMU_TEST_FLAKY_TESTS'), 'Test sometimes gets stuck')
     def test_x86_64_pc(self):
         """
         :avocado: tags=arch:x86_64

tests/avocado/reverse_debugging.py

@@ -12,6 +12,7 @@
 
 from avocado import skipIf
 from avocado_qemu import BUILD_DIR
+from avocado.utils import datadrainer
 from avocado.utils import gdb
 from avocado.utils import process
 from avocado.utils.network.ports import find_free_port
@@ -52,6 +53,10 @@ def run_vm(self, record, shift, args, replay_path, image_path, port):
         if args:
             vm.add_args(*args)
         vm.launch()
+        console_drainer = datadrainer.LineLogger(vm.console_socket.fileno(),
+                                    logger=self.log.getChild('console'),
+                                    stop_check=(lambda : not vm.is_running()))
+        console_drainer.start()
         return vm
 
     @staticmethod
@@ -200,8 +205,6 @@ def get_pc(self, g):
         return self.get_reg_le(g, self.REG_PC) \
             + self.get_reg_le(g, self.REG_CS) * 0x10
 
-    # unidentified gitlab timeout problem
-    @skipIf(os.getenv('GITLAB_CI'), 'Running on GitLab')
     def test_x86_64_pc(self):
         """
         :avocado: tags=arch:x86_64
@@ -217,8 +220,6 @@ class ReverseDebugging_AArch64(ReverseDebugging):
 
     REG_PC = 32
 
-    # unidentified gitlab timeout problem
-    @skipIf(os.getenv('GITLAB_CI'), 'Running on GitLab')
     def test_aarch64_virt(self):
         """
         :avocado: tags=arch:aarch64
@@ -241,8 +242,6 @@ class ReverseDebugging_ppc64(ReverseDebugging):
 
     REG_PC = 0x40
 
-    # unidentified gitlab timeout problem
-    @skipIf(os.getenv('GITLAB_CI'), 'Running on GitLab')
     def test_ppc64_pseries(self):
         """
         :avocado: tags=arch:ppc64
@@ -254,7 +253,6 @@ def test_ppc64_pseries(self):
         self.endian_is_le = False
         self.reverse_debugging()
 
-    @skipIf(os.getenv('GITLAB_CI'), 'Running on GitLab')
     def test_ppc64_powernv(self):
         """
         :avocado: tags=arch:ppc64

tests/avocado/version.py

@@ -22,4 +22,4 @@ def test_qmp_human_info_version(self):
         self.vm.launch()
         res = self.vm.cmd('human-monitor-command',
                           command_line='info version')
-        self.assertRegexpMatches(res, r'^(\d+\.\d+\.\d)')
+        self.assertRegex(res, r'^(\d+\.\d+\.\d)')

tests/avocado/virtio-gpu.py

@@ -149,10 +149,8 @@ def test_vhost_user_vga_virgl(self):
             # TODO: probably fails because we are missing the VirGL features
             self.cancel("VirGL not enabled?")
         self.wait_for_console_pattern("as init process")
-        exec_command_and_wait_for_pattern(
-            self, "/usr/sbin/modprobe virtio_gpu", ""
-        )
-        self.wait_for_console_pattern("features: +virgl -edid")
+        exec_command_and_wait_for_pattern(self, "/usr/sbin/modprobe virtio_gpu",
+                                          "features: +virgl +edid")
         self.vm.shutdown()
         qemu_sock.close()
         vugp.terminate()