Add a CPU usage check for HAWK/PUMA

This adds a CPU usage check on the ha/check_hawk test module while a client running the ha/hawk_gui test module is interacting with HAWK. It will soft fail with bsc#1179609 (HAWK/PUMA consume a considerable amount of CPU) if HAWK/PUMA CPU usage is over 50%.
os-autoinst · May 3, 2021 · 8bb84b0 · 8bb84b0
1 parent ba6b709
commit 8bb84b0
Show file tree

Hide file tree

Showing 3 changed files with 45 additions and 3 deletions.
diff --git a/tests/ha/barrier_init.pm b/tests/ha/barrier_init.pm
@@ -108,9 +108,11 @@ sub run {
         barrier_create("PACEMAKER_CTS_CHECKED_$cluster_name", $num_nodes + 1);
 
         # HAWK_GUI_ barriers also have to wait in the client
-        barrier_create("HAWK_GUI_INIT_$cluster_name",    $num_nodes + 1);
-        barrier_create("HAWK_GUI_CHECKED_$cluster_name", $num_nodes + 1);
-        barrier_create("HAWK_FENCE_$cluster_name",       $num_nodes + 1);
+        barrier_create("HAWK_GUI_INIT_$cluster_name",            $num_nodes + 1);
+        barrier_create("HAWK_GUI_CHECKED_$cluster_name",         $num_nodes + 1);
+        barrier_create("HAWK_GUI_CPU_TEST_START_$cluster_name",  $num_nodes + 1);
+        barrier_create("HAWK_GUI_CPU_TEST_FINISH_$cluster_name", $num_nodes + 1);
+        barrier_create("HAWK_FENCE_$cluster_name",               $num_nodes + 1);
 
         # CTDB barriers
         barrier_create("CTDB_INIT_$cluster_name", $num_nodes + 1);

diff --git a/tests/ha/check_hawk.pm b/tests/ha/check_hawk.pm
@@ -19,6 +19,43 @@ use lockapi;
 use hacluster qw(get_cluster_name is_node);
 use utils 'systemctl';
 use version_utils 'is_sle';
+use List::Util qw(sum);
+
+sub check_hawk_cpu {
+    my $cluster_name = get_cluster_name;
+    my @cpu_usage    = ();
+
+    barrier_wait("HAWK_GUI_CPU_TEST_START_$cluster_name");
+    while (!barrier_try_wait("HAWK_GUI_CPU_TEST_FINISH_$cluster_name")) {
+        # Wrapping script_output in eval { } as node can be fenced by hawk test from client.
+        # In fenced node, script_output will croak and kill the test. This prevents it
+        my $metric = eval {
+            script_output q@ps axo pcpu,comm | awk '/hawk|puma/ {total += $1} END {print "cpu_usage["total"]"}'@,
+              proceed_on_failure => 1;
+        };
+        if ($@) {
+            # When script_output croaks, command may be typed when SUT is on the grub menu
+            # and either boot the system or get into grub editing. If system has booted,
+            # force a new fence; if it's still in grub menu, do nothing; otherwise send an
+            # ESC to return SUT to grub menu and exit the loop
+            if (check_screen('linux-login')) {
+                reset_consoles;
+                select_console('root-console');
+                enter_cmd 'echo b > /proc/sysrq-trigger';
+            }
+            else {
+                send_key 'esc' unless check_screen('grub2');
+            }
+            barrier_wait("HAWK_GUI_CPU_TEST_FINISH_$cluster_name");
+            last;
+        }
+        push @cpu_usage, $metric =~ /cpu_usage\[([\d\.]+)\]/;
+        sleep bmwqemu::scale_timeout(1);
+    }
+    my $cpu_usage = sum(@cpu_usage) / @cpu_usage;
+    record_info "CPU usage", "HAWK/PUMA CPU usage was $cpu_usage";
+    record_soft_failure "bsc#1179609 - HAWK/PUMA consume a considerable amount of CPU" if ($cpu_usage >= 50);
+}
 
 sub run {
     my $cluster_name = get_cluster_name;
@@ -55,6 +92,7 @@ sub run {
     # If testing HAWK GUI, also wait for those barriers
     if (get_var('HAWKGUI_TEST_ROLE')) {
         barrier_wait("HAWK_GUI_INIT_$cluster_name");
+        check_hawk_cpu;
         barrier_wait("HAWK_GUI_CHECKED_$cluster_name");
     }
 

diff --git a/tests/ha/hawk_gui.pm b/tests/ha/hawk_gui.pm
@@ -86,6 +86,7 @@ sub run {
     add_to_known_hosts($node2);
     assert_script_run "mkdir -m 1777 $path";
     assert_script_run "xhost +";
+    barrier_wait("HAWK_GUI_CPU_TEST_START_$cluster_name");
     my $docker_cmd = "docker run --rm --name test --ipc=host -v /tmp/.X11-unix:/tmp/.X11-unix -e DISPLAY=\$DISPLAY -v \$PWD/$path:/$path ";
     $docker_cmd .= "$docker_image -b $browser -H $node1 -S $node2 -s $testapi::password -r /$results --virtual-ip $virtual_ip";
     enter_cmd "$docker_cmd | tee $logs; echo $pyscr-\$PIPESTATUS > $retcode";
@@ -114,6 +115,7 @@ sub run {
     save_screenshot;
 
     assert_screen "generic-desktop";
+    barrier_wait("HAWK_GUI_CPU_TEST_FINISH_$cluster_name");
 
     # Error, log and results handling
     select_console 'user-console';