test: Robustify NM checkpoint rollback tests

At the beginning of the test, the VM is very busy as so many things
happen at once. This causes NetworkManager device updates to be very
slow and run into the checkpoint revert threshold. This causes the new
settings for eth2 to never be applied, and instead the test gets stuck
on the "Change/Rollback" dialog.

To mitigate this, let the VM settle down a bit.

Fixes #13464
Closes #13545

test/common/testlib.py

@@ -1131,6 +1131,20 @@ def copy_cores(self, title, label=None):
                         print("Core dumps downloaded to %s" % (dest))
                         attach(dest)
 
+    def settle_cpu(self):
+        '''Wait until CPU usage in the VM settles down
+
+        Wait until the process with the highest CPU usage drops below 20%
+        usage. Wait for up to a minute, then return. There is no error if the
+        CPU stays busy, as usually a test then should just try to run anyway.
+        '''
+        for retry in range(20):
+            # get the CPU percentage of the most busy process
+            busy_proc = self.machine.execute("ps --no-headers -eo pcpu,pid,args | sort -k 1 -n -r | head -n1")
+            if float(busy_proc.split()[0]) < 20.0:
+                break
+            time.sleep(3)
+
 
 def jsquote(str):
     return json.dumps(str)

test/verify/check-networking-checkpoints

@@ -117,6 +117,9 @@ class TestNetworking(NetworkCase):
         b.click("#networking-interfaces tr[data-interface='%s']" % iface)
         b.wait_visible("#network-interface")
 
+        # checkpoints are realtime sensitive, avoid long NM operations
+        self.settle_cpu()
+
         self.slow_down_dhclient(dhcp_delay)
 
         # Disconnect and trigger a slow rollback

test/verify/check-networking-settings

@@ -38,6 +38,8 @@ class TestNetworking(NetworkCase):
 
         iface = self.add_iface(activate=False)
         self.wait_for_iface(iface, active=False)
+        # checkpoints are realtime sensitive, avoid long NM operations
+        self.settle_cpu()
 
         b.click("#networking-interfaces tr[data-interface='%s']" % iface)
         b.wait_visible("#network-interface")