Add support for backing up disks during test

OpenQA/Isotovideo/Interface.pm

@@ -9,7 +9,7 @@ use Mojo::Base -strict, -signatures;
 # -> increment on every change of such APIs
 # -> never move that variable to another place (when refactoring)
 #    because it may be accessed by the tests itself
-our $version = 39;
+our $version = 40;
 
 # major version of the (web socket) API relevant to the developer mode
 # -> increment when making non-backward compatible changes to that API

backend/baseclass.pm

@@ -488,6 +488,8 @@ sub switch_network ($self, $args) { $self->notimplemented }
 
 sub save_memory_dump ($self, $args) { $self->notimplemented }
 
+sub save_storage ($self, $args) { $self->notimplemented }
+
 ## MAY be overwritten:
 
 # vm's would return

backend/qemu.pm

@@ -9,6 +9,7 @@ use File::Basename 'dirname';
 use File::Path 'mkpath';
 use File::Which;
 use Time::HiRes qw(sleep gettimeofday);
+use Time::Seconds;
 use IO::Socket::UNIX 'SOCK_STREAM';
 use IO::Handle;
 use POSIX qw(strftime :sys_wait_h mkfifo);
@@ -346,6 +347,55 @@ sub deflate_balloon ($self) {
     $self->handle_qmp_command({execute => 'balloon', arguments => {value => $ram_bytes}}, fatal => 1);
 }
 
+sub save_storage ($self, $args) {
+    my $vars = \%bmwqemu::vars;
+    my $bdc = $self->{proc}->blockdev_conf;
+    my $fname = $args->{filename};
+    my $rsp = $self->handle_qmp_command({execute => 'query-status'}, fatal => 1);
+    bmwqemu::diag("Saving storage devices (current VM state is $rsp->{return}->{status})");
+    my $was_running = $rsp->{return}->{status} eq 'running';
+    if ($was_running) {
+        $self->inflate_balloon();
+        $self->freeze_vm();
+    }
+    mkpath("assets_public");
+    $bdc->for_each_drive(sub ($drive) {
+            my $size = $drive->{drive}->{size};
+            my $id = "$drive->{id}-backup-$fname";
+            my $node = $drive->{drive}->{node_name};
+            # no need to save CDs
+            return if ($node =~ qr/cd[0-9]-overlay/);
+            my $my_node = "$node-$fname";
+            my $bck_file = "assets_public/$my_node-$vars->{NAME}.qcow2";
+            # create disk
+            runcmd('qemu-img', 'create', '-f', 'qcow2', "$bck_file", $size);
+            my $req = {execute => 'blockdev-add',
+                arguments => {driver => 'qcow2', 'node-name' => $my_node,
+                    file => {driver => 'file', filename => $bck_file}
+                }};
+            $self->handle_qmp_command($req, fatal => 1);
+            $req = {execute => 'blockdev-backup',
+                arguments => {device => $node, target => $my_node,
+                    sync => 'full', 'job-id' => $id}};
+            $self->handle_qmp_command($req, fatal => 1);
+            my $return;
+            my $timeout = $vars->{SAVE_STORAGE_TIMEOUT} // (ONE_MINUTE * 15);
+            # wait for background job to finish before we continue
+            do {
+                die "Saving volume $node exceeded the timeout" if $timeout == 0;
+                my $query = {execute => 'query-jobs'};
+                $return = $self->handle_qmp_command($query, fatal => 1)->{return};
+                sleep 1;
+                --$timeout;
+            } while (@$return);
+    });
+    bmwqemu::diag("Saving storage complete");
+    if ($was_running) {
+        $self->cont_vm();
+        $self->deflate_balloon();
+    }
+}
+
 sub save_snapshot ($self, $args) {
     my $vars = \%bmwqemu::vars;
     my $vmname = $args->{name};

doc/backend_vars.asciidoc

@@ -200,6 +200,7 @@ WORKER_CLASS;string;undef;qemu system types
 WORKER_HOSTNAME;string;undef;Worker hostname
 QEMU_VIDEO_DEVICE;string;virtio-gpu-pci on ARM, VGA otherwise;Video device to use with VM (using -device, not -vga). Can have options appended e.g. "virtio-gpu-gl,edid=on", but it is better to set QEMU_VIDEO_DEVICE_OPTIONS. See qemu docs and https://www.kraxel.org/blog/2019/09/display-devices-in-qemu/ for valid choices
 QEMU_VIDEO_DEVICE_OPTIONS;string;none;Additional options for QEMU_VIDEO_DEVICE (comma-separated). Will be appended after automatically-generated resolution setting options on devices that support EDID
+SAVE_STORAGE_TIMEOUT;integer;900;Timeout for saving one storage volume in `save_storage` test API function.
 |====================
 
 .SVIRT backend

doc/memorydumps.asciidoc

@@ -8,11 +8,11 @@ IMPORTANT: Currently only the QEMU backend is supported.
 
 == How to use
 
-Two methods are available to the test writer, `freeze_vm` and `save_memory_dump`
+Three methods are available to the test writer, `freeze_vm`, `save_memory_dump` and `save_storage`.
 
 The feature is enabled by calling the aforementioned methods, using
-a https://github.com/os-autoinst/openQA/blob/master/docs/WritingTests.asciidoc#how-to-write-tests[post_fail_hook] and setting the test's flags as fatal. The `save_memory_dump` method can be however
-called at any point within the test.
+a https://github.com/os-autoinst/openQA/blob/master/docs/WritingTests.asciidoc#how-to-write-tests[post_fail_hook] and setting the test's flags as fatal.
+The `save_memory_dump`  and `save_storage` methods can be however called at any point within the test.
 
 === save_memory_dump
 
@@ -29,6 +29,15 @@ without any risk of data changing.
 
 NOTE: Call this method to ensure memory and disk dump refer to the same machine state.
 
+
+=== save_storage
+
+If it is supported by the backend `save_storage` will save all storage devices into backup.
+This can be called from anywhere within a test. For multiple calls within a test module
+a filename needs to be specified. By default it uses the test module name.
+
+NOTE: This method is time-consuming and requires a lot of space.
+
 == Examples
 
 A very simple way to use this helpful feature is the following:
@@ -42,3 +51,8 @@ sub post_fail_hook ($self) {
 
 sub test_flags ($) { { fatal => 1 } }
 ----------------------------------------
+
+[source,perl]
+----------------------------------------
+    save_storage(filename => 'my-backup');
+----------------------------------------

t/03-testapi.t

@@ -1090,6 +1090,7 @@ subtest 'assert/check recorded sound' => sub {
 
 lives_ok { power('on') } 'power can be called';
 lives_ok { save_memory_dump } 'save_memory_dump can be called';
+lives_ok { save_storage } 'save_storage can be called';
 lives_ok { freeze_vm } 'freeze_vm can be called';
 lives_ok { resume_vm } 'resume_vm can be called';
 

t/04-check_vars_docu.t

@@ -23,7 +23,7 @@ use constant VARS_DOC => DOC_DIR . '/backend_vars.asciidoc';
 my @backend_blocklist = qw();
 # blocklist of vars per backend. These vars will be ignored during vars exploration
 my %var_blocklist = (
-    QEMU => ['WORKER_ID', 'WORKER_INSTANCE'],
+    QEMU => ['WORKER_ID', 'WORKER_INSTANCE', 'NAME'],
     VAGRANT => ['QEMUCPUS', 'QEMURAM'],
     GENERALHW => ['HDD_1'],
     SVIRT => ['JOBTOKEN'],

t/18-backend-qemu.t

@@ -486,6 +486,67 @@ subtest 'snapshot handling' => sub {
     ], 'expected QMP commands invoked when loading snapshot' or diag explain $$invoked_qmp_cmds;
 };
 
+subtest 'save storage' => sub {
+    $bmwqemu::vars{QEMU_BALLOON_TARGET} = undef;
+    $bmwqemu::vars{NAME} = 'FAKE_TEST';
+    my $i = 0;
+    my %running = (return => {status => 'running'});
+    my %done = (return => []);
+    my @fake_qmp_answer = (\%running, \%done, \%done, \%done, \%done, \%done, \%done, \%done);
+    $backend_mock->unmock('handle_qmp_command');
+    $mock_bmwqemu->unmock('diag');
+    $backend_mock->redefine(handle_qmp_command => sub { push @{$called{handle_qmp_command}}, $_[1]; $fake_qmp_answer[$i++] });
+    $$invoked_qmp_cmds = undef;
+    combined_like { $backend->save_storage({filename => 'fakevm'}) } qr/Saving storage complete/i, 'completion logged (1)';
+    is_deeply $$invoked_qmp_cmds, [
+        {execute => 'query-status'},
+        {execute => 'stop'},
+        {arguments => {
+                driver => 'qcow2',
+                file => {
+                    driver => 'file',
+                    filename => 'assets_public/hd0-overlay2-fakevm-FAKE_TEST.qcow2'
+                },
+                'node-name' => 'hd0-overlay2-fakevm'
+            },
+            execute => 'blockdev-add'
+        },
+        {arguments => {
+                device => 'hd0-overlay2',
+                'job-id' => 'hd0-backup-fakevm',
+                sync => 'full',
+                target => 'hd0-overlay2-fakevm'
+            },
+            execute => 'blockdev-backup'
+        },
+        {execute => 'query-jobs'},
+        {arguments => {
+                driver => 'qcow2',
+                file => {
+                    driver => 'file',
+                    filename => 'assets_public/some-id-overlay2-fakevm-FAKE_TEST.qcow2'
+                },
+                'node-name' => 'some-id-overlay2-fakevm'
+            },
+            execute => 'blockdev-add'
+        },
+        {arguments => {
+                device => 'some-id-overlay2',
+                'job-id' => 'some-id-backup-fakevm',
+                sync => 'full',
+                target => 'some-id-overlay2-fakevm'
+            },
+            execute => 'blockdev-backup'},
+        {execute => 'query-jobs'},
+        {execute => 'cont'}], 'excepted QMP commands when saving storage' or diag explain $$invoked_qmp_cmds;
+    # timeout exceeded
+    $bmwqemu::vars{SAVE_STORAGE_TIMEOUT} = 2;
+    $i = 0;
+    my %working = (return => [{s => 'r'}]);
+    @fake_qmp_answer = (\%running, \%working, \%working, \%working, \%working, \%working, \%working, \%working);
+    combined_like { throws_ok { $backend->save_storage({filename => 'failvm'}) } qr/Saving volume hd0-overlay2 exceeded the timeout/, 'die on timeout exceeed' } qr/current VM state is running/, 'exception happended in save_storage sub';
+};
+
 subtest 'special cases when starting QEMU' => sub {
     # set certain variables to test special handling for them that is not otherwise tested
     $bmwqemu::scriptdir = "$Bin/..";    # for dmi data
@@ -513,7 +574,6 @@ subtest 'special cases when starting QEMU' => sub {
     $backend_mock->redefine(_child_process => sub ($self, $coderef) { ++$pid });
     $proc->redefine(load_state => sub ($self) { ++$load_state });
     $proc->redefine(static_param => sub ($self, @params) { push @qemu_params, @params });
-    $mock_bmwqemu->unmock('diag');
     $backend_mock->redefine(requires_audiodev => 0);
 
     my @invoked_cmds;

t/23-baseclass.t

@@ -83,6 +83,7 @@ subtest 'not implemented' => sub {
         [do_extract_assets => 23],
         [switch_network => 23],
         [save_memory_dump => 23],
+        [save_storage => 23]
     );
     for my $test (@tests) {
         my ($m, @args) = @$test;

t/99-full-stack.t

@@ -8,7 +8,7 @@ use Mojo::Base -strict, -signatures;
 
 use FindBin '$Bin';
 use lib "$Bin/../external/os-autoinst-common/lib";
-use OpenQA::Test::TimeLimit '300';
+use OpenQA::Test::TimeLimit '450';
 use Test::Warnings ':report_warnings';
 use Try::Tiny;
 use File::Basename;
@@ -44,6 +44,7 @@ path('vars.json')->spurt(<<EOV);
    "VERSION" : "1",
    "SSH_CONNECT_RETRY"  : "2",
    "SSH_CONNECT_RETRY_INTERVAL"  : ".001",
+   "NAME" : "00001-1-i386@32bit",
 }
 EOV
 # create screenshots
@@ -62,6 +63,8 @@ like $log, qr/wait_still_screen: detected same image for 1 seconds/, 'test type
 like $log, qr/wait_still_screen: detected same image for 0\.1 seconds/, 'test type string and wait for .1 seconds';
 like $log, qr/.*event.*STOP/, 'Machine properly paused';
 like $log, qr/.*event.*RESUME/, 'Machine properly resumed';
+like $log, qr/Saving storage devices \(current VM state is running\)/, 'save_storage started';
+like $log, qr/Saving storage complete/, 'save_storage done';
 like $log, qr/get_test_data returned expected file/, 'get_test_data test';
 like $log, qr/save_tmp_file returned expected file/, 'save_tmp_file test';
 unlike $log, qr/warn.*qemu-system.*terminating/, 'No warning about expected termination';

t/data/tests/main.pm

@@ -37,6 +37,7 @@ unless ($integration_tests) {
 autotest::loadtest "tests/boot.pm";
 autotest::loadtest "tests/assert_screen.pm";
 unless ($integration_tests) {
+    autotest::loadtest "tests/save_storage.pm";
     autotest::loadtest "tests/typing.pm";
     autotest::loadtest "tests/select_console_fail_test.pm";
     autotest::loadtest "tests/select_ssh_console_fail_test.pm";

t/data/tests/tests/save_storage.pm

@@ -0,0 +1,14 @@
+# Copyright SUSE LLC
+# SPDX-License-Identifier: GPL-2.0-or-later
+
+use Mojo::Base 'basetest', -signatures;
+use testapi;
+
+
+sub run ($) {
+    save_storage;
+}
+
+sub test_flags ($) { {fatal => 1} }
+
+1;

testapi.pm

@@ -53,7 +53,7 @@ our @EXPORT = qw($realname $username $password $serialdev %cmd %vars
   become_root x11_start_program ensure_installed eject_cd power
 
   switch_network
-  save_memory_dump freeze_vm resume_vm
+  save_memory_dump freeze_vm resume_vm save_storage
 
   diag hashed_string
 
@@ -1929,6 +1929,23 @@ sub save_memory_dump (%nargs) {
     query_isotovideo('backend_save_memory_dump', \%nargs);
 }
 
+=head2 save_storage
+
+  save_storage(filename => undef);
+
+Saves all of the SUT volumes using C<$filename> as part of the final filename,
+the default will be the current test's name.
+
+I<Currently only qemu backend is supported.>
+
+=cut
+
+sub save_storage (%nargs) {
+    $nargs{filename} ||= $autotest::current_test->{name};
+    query_isotovideo('backend_save_storage', \%nargs);
+}
+
+
 =head2 freeze_vm
 
   freeze_vm;