use null backend 4real

t/14-isotovideo.t

@@ -20,13 +20,13 @@ use OpenQA::Isotovideo::Utils qw(checkout_wheels handle_generated_assets load_te
 use OpenQA::Isotovideo::CommandHandler;
 
 # Each full isotovideo invocation (not using _exit_after_schedule=1)
-# is expected to take maximum ~6 seconds per run.
+# is expected to take maximum ~6.67 seconds per run.
 #
-# There are 5 invocations, that approxs to 30 seconds.
+# There are 3 invocations, that approxs to ~20 seconds.
 #
 # Update this number if a new test involving a full isotovideo invocation is
 # added.
-use OpenQA::Test::TimeLimit '30';
+use OpenQA::Test::TimeLimit '20';
 
 my $dir = tempdir("/tmp/$FindBin::Script-XXXX");
 my $toplevel_dir = abs_path(dirname(__FILE__) . '/..');
@@ -247,48 +247,61 @@ subtest 'productdir variable relative/absolute' => sub {
 };
 
 subtest 'exit status from test results: no test scheduled' => sub {
-    # full isotovideo invocation
+    # dummy isotovideo invocation
     chdir($pool_dir);
     path(bmwqemu::STATE_FILE)->remove if -e bmwqemu::STATE_FILE;
     path('vars.json')->remove if -e 'vars.json';
-    my $module = 'tests/failing_module';
+    # TODO: backend/baseclass.pm does not respect backend when reading
+    #       the serial console.
+    path('serial0')->touch();
+    my $module_basename = 'failing_module';
+    my $module = "tests/$module_basename";
     my $log = combined_from { isotovideo(
             default_opts => '--exit-status-from-test-results backend=null',
-            opts => "casedir=$data_dir/empty_test_distribution qemu_no_kvm=1 arch=i386 backend=qemu qemu=i386 novideo=1", exit_code => 100) };
+            opts => "casedir=$data_dir/tests schedule=$module", exit_code => 101) };
+
+    like $log, qr/scheduling $module_basename $module\.pm/, 'module scheduled';
+    like $log, qr/Test result \[testresults\/result\-$module_basename\.json\] fail/, 'module test result parsed';
 };
 
 subtest 'exit status from test results: tests scheduled' => sub {
-    # full isotovideo invocation
+    # dummy isotovideo invocation
     chdir($pool_dir);
     path(bmwqemu::STATE_FILE)->remove if -e bmwqemu::STATE_FILE;
     path('vars.json')->remove if -e 'vars.json';
+    # TODO: backend/baseclass.pm does not respect backend when reading
+    #       the serial console.
+    path('serial0')->touch();
     my $module_basename = 'failing_module';
     my $module = "tests/$module_basename";
     my $log = combined_from { isotovideo(
             default_opts => '--exit-status-from-test-results backend=null',
-            opts => "casedir=$data_dir/tests schedule=$module qemu_no_kvm=1 arch=i386 backend=qemu qemu=i386 novideo=1", exit_code => 101) };
+            opts => "casedir=$data_dir/tests schedule=$module", exit_code => 101) };
 
     like $log, qr/scheduling $module_basename $module\.pm/, 'module scheduled';
     like $log, qr/Test result \[testresults\/result\-$module_basename\.json\] fail/, 'module test result parsed';
 };
 
 subtest 'exit status from test results: softfailed test' => sub {
-    # full isotovideo invocation
+    # dummy isotovideo invocation
     chdir($pool_dir);
     path(bmwqemu::STATE_FILE)->remove if -e bmwqemu::STATE_FILE;
     path('vars.json')->remove if -e 'vars.json';
+    # TODO: backend/baseclass.pm does not respect backend when reading
+    #       the serial console.
+    path('serial0')->touch();
     my $module_basename = 'soft_failed_test';
     my $module = "tests/$module_basename";
     my $log = combined_from { isotovideo(
             default_opts => '--exit-status-from-test-results backend=null',
-            opts => "casedir=$data_dir/tests schedule=$module qemu_no_kvm=1 arch=i386 backend=qemu qemu=i386 novideo=1", exit_code => 0) };
+            opts => "casedir=$data_dir/tests schedule=$module", exit_code => 0) };
 
     like $log, qr/scheduling $module_basename $module\.pm/, 'module scheduled';
     like $log, qr/Test result \[testresults\/result\-$module_basename\.json\] softfail/, 'module test result parsed';
 };
 
 subtest 'upload assets on demand even in failed jobs' => sub {
-    # full isotovideo invocation
+    # qemu isotovideo invocation
     chdir($pool_dir);
     path(bmwqemu::STATE_FILE)->remove if -e bmwqemu::STATE_FILE;
     path('vars.json')->remove if -e 'vars.json';
@@ -302,7 +315,7 @@ subtest 'upload assets on demand even in failed jobs' => sub {
 };
 
 subtest 'load test success when casedir and productdir are relative path' => sub {
-    # full isotovideo invocation
+    # qemu isotovideo invocation
     chdir($pool_dir);
     path(bmwqemu::STATE_FILE)->remove if -e bmwqemu::STATE_FILE;
     path('vars.json')->remove if -e 'vars.json';
@@ -407,6 +420,7 @@ subtest 'publish assets' => sub {
 done_testing();
 
 END {
+    rmtree "./serial0";
     rmtree "$Bin/data/tests/product";
     rmtree "$data_dir/wheels_dir/writer";
     rmtree "$pool_dir/writer";