Add variable SCHEDULE to define the full schedule by variable

Also allows to start tests without relying on a vars.json file at all. Moves the test variable sanity checks together for easier separation. The test variables "CASEDIR" and "PRJDIR" were sanity-checked at two different locations, in isotovideo and bmwqemu.pm, where not actually used. Moving the checks from bmwqemu.pm to isotovideo allows to set mandatory variables by command line parameters
os-autoinst · Sep 17, 2018 · 32c4e65 · 32c4e65
1 parent 53731b6
commit 32c4e65
Show file tree

Hide file tree

Showing 7 changed files with 87 additions and 43 deletions.
diff --git a/bmwqemu.pm b/bmwqemu.pm
@@ -45,7 +45,8 @@ sub mydie;
 $| = 1;
 
 
-our $default_timeout = 30;    # assert timeout, 0 is a valid timeout
+our $default_timeout      = 30;                        # assert timeout, 0 is a valid timeout
+our $openqa_default_share = '/var/lib/openqa/share';
 
 my @ocrrect;
 
@@ -71,7 +72,9 @@ sub load_vars {
     my $fn  = "vars.json";
     my $ret = {};
     local $/;
-    open(my $fh, '<', $fn) or return 0;
+    my $fh;
+    eval { open($fh, '<', $fn) };
+    return 0 if $@;
     eval { $ret = JSON->new->relaxed->decode(<$fh>); };
     die "parse error in vars.json:\n$@" if $@;
     close($fh);
@@ -130,29 +133,15 @@ sub init {
             return sprintf(strftime("[%FT%T.%%04d %Z] [$level] ", localtime($time)), 1000 * ($time - int($time))) . join("\n", @lines, '');
 
         });
+}
 
-    die "CASEDIR variable not set in vars.json, unknown test case directory" if !$vars{CASEDIR};
-
-    unless ($vars{PRJDIR}) {
-        if (index($vars{CASEDIR}, '/var/lib/openqa/share') != 0) {
-            die "PRJDIR not specified and CASEDIR ($vars{CASEDIR}) does not appear to be a
-                subdir of default (/var/lib/openqa/share). Please specify PRJDIR in vars.json";
-        }
-        $vars{PRJDIR} = '/var/lib/openqa/share';
-    }
-
-
+sub ensure_valid_vars {
     # defaults
     $vars{QEMUPORT} ||= 15222;
     $vars{VNC}      ||= 90;
     # openQA already sets a random string we can reuse
     $vars{JOBTOKEN} ||= random_string(10);
 
-    save_vars();
-
-    ## env vars end
-
-    ## some var checks
     if ($gocrbin && !-x $gocrbin) {
         $gocrbin = undef;
     }
@@ -162,6 +151,18 @@ sub init {
         }
     }
 
+    die "CASEDIR variable not set, unknown test case directory" if !defined $vars{CASEDIR};
+    die "No scripts in $vars{CASEDIR}" if !-e "$vars{CASEDIR}";
+    unless ($vars{PRJDIR}) {
+        if (index($vars{CASEDIR}, $openqa_default_share) != 0) {
+            diag("PRJDIR not specified and CASEDIR ($vars{CASEDIR}) does not appear to be a subdir of default ($openqa_default_share), using CASEDIR instead");
+            $vars{PRJDIR} = $vars{CASEDIR};
+        }
+        else {
+            $vars{PRJDIR} = $openqa_default_share;
+        }
+    }
+    save_vars();
 }
 
 ## some var checks end

diff --git a/doc/backend_vars.asciidoc b/doc/backend_vars.asciidoc
@@ -8,6 +8,7 @@ Supported variables per backend
 Variable;Values allowed;Default value;Explanation
 INCLUDE_MODULES;string;;comma separated names or fullnames of test modules to be included while excluding all that do not match, e.g. "boot,mod1"
 EXCLUDE_MODULES;string;;comma separated names or fullnames of test modules to exclude. Can be combined with INCLUDE_MODULES and has precedence, e.g. to additionally exclude modules based on an include-list
+SCHEDULE;string;;comma separated list of relative paths to test modules within CASEDIR without the implicit prefix folder 'tests/' and without the file extension '.pm' to be scheduled instead of evaluating a schedule from the test distributions main.pm file, e.g. "boot,console/mod1"
 _EXIT_AFTER_SCHEDULE;boolean;0;Exit test execution immediately after evaluation of the test schedule, e.g. to check only which test modules would be executed
 _SKIP_POST_FAIL_HOOKS;boolean;0;Skip the execution of post_fail_hook methods if set. This can be useful to save test execution time during test development when the post_fail_hook is not expected to provide any value as most likely the test developer already knows what needs to be done as a next step on a test fail.
 NOVIDEO;boolean;0;Do not encode video if set

diff --git a/isotovideo b/isotovideo
@@ -125,18 +125,14 @@ $| = 1;
 
 $bmwqemu::scriptdir = $installprefix;
 bmwqemu::init();
-
 for my $arg (@ARGV) {
     if ($arg =~ /^([[:alnum:]_\[\]\.]+)=(.+)/) {
         my $key = uc $1;
         $bmwqemu::vars{$key} = $2;
         diag("Setting forced test parameter $key -> $2");
     }
 }
-
-# Sanity checks
-die "CASEDIR environment variable not set, unknown test case directory" if !defined $bmwqemu::vars{CASEDIR};
-die "No scripts in $bmwqemu::vars{CASEDIR}" if !-e "$bmwqemu::vars{CASEDIR}";
+bmwqemu::ensure_valid_vars();
 
 my $cmd_srv_process;
 my $testprocess;
@@ -225,14 +221,22 @@ $testapi::distri = distribution->new;
 # further dependencies (the tests get it through autotest)
 my @oldINC = @INC;
 unshift @INC, $bmwqemu::vars{CASEDIR} . '/lib';
-require $bmwqemu::vars{PRODUCTDIR} . "/main.pm";
+if ($bmwqemu::vars{SCHEDULE}) {
+    diag 'Enforced test schedule by \'SCHEDULE\' variable in action';
+    autotest::loadtest('tests/' . $_ . '.pm') foreach split(',', $bmwqemu::vars{SCHEDULE});
+}
+elsif (-e $bmwqemu::vars{PRODUCTDIR} . '/main.pm') {
+    require $bmwqemu::vars{PRODUCTDIR} . '/main.pm';
+}
+else {
+    die '\'SCHEDULE\' not set and ' . $bmwqemu::vars{PRODUCTDIR} . '/main.pm not found, need one of both';
+}
 @INC = @oldINC;
 
 if ($bmwqemu::vars{_EXIT_AFTER_SCHEDULE}) {
     diag 'Early exit has been requested with _EXIT_AFTER_SCHEDULE. Only evaluating test schedule.';
     exit 0;
 }
-
 testapi::init();
 
 # init part

diff --git a/t/12-bmwqemu.t b/t/12-bmwqemu.t
@@ -19,13 +19,18 @@ use 5.018;
 use warnings;
 use Test::More;
 use File::Temp 'tempdir';
+use File::Basename;
 use File::Path 'make_path';
+use Cwd 'abs_path';
 use JSON;
 
 BEGIN {
     unshift @INC, '..';
 }
 
+my $toplevel_dir = abs_path(dirname(__FILE__) . '/..');
+my $data_dir     = "$toplevel_dir/t/data";
+
 sub create_vars {
     my $data = shift;
     open(my $varsfh, '>', 'vars.json') || BAIL_OUT('can not create vars.json');
@@ -51,8 +56,9 @@ subtest 'CASEDIR is mandatory' => sub {
     eval {
         use bmwqemu ();
         bmwqemu::init;
+        bmwqemu::ensure_valid_vars();
     };
-    like($@, qr(CASEDIR variable not set in vars.json.*), 'bmwqemu refuses to init');
+    like($@, qr(CASEDIR variable not set.*), 'bmwqemu refuses to init');
 
 
     my %vars = %{read_vars()};
@@ -61,36 +67,39 @@ subtest 'CASEDIR is mandatory' => sub {
     is($vars{PRJDIR}, $dir, 'PRJDIR unchanged');
 };
 
-subtest 'test PRJDIR default' => sub {
-    my $dir = '/var/lib/openqa/share/tests/test';
+subtest 'test CASEDIR not under PRJDIR default' => sub {
+    my $dir = "$data_dir/tests";
     create_vars({CASEDIR => $dir});
 
     eval {
         use bmwqemu ();
         bmwqemu::init;
+        bmwqemu::ensure_valid_vars();
     };
     ok(!$@, 'init successful');
 
     my %vars = %{read_vars()};
     ok(!$vars{DISTRI}, 'DISTRI not supplied and not set');
     is($vars{CASEDIR}, $dir, 'CASEDIR unchanged');
-    is($vars{PRJDIR}, '/var/lib/openqa/share', 'PRJDIR set to default');
+    is($vars{PRJDIR},  $dir, 'PRJDIR set CASEDIR');
 };
 
-subtest 'test CASEDIR not under PRJDIR default' => sub {
-    my $dir = '/tmp/some/dir/tests/test';
+subtest 'test PRJDIR default' => sub {
+    my $dir = "$data_dir/tests";
     create_vars({CASEDIR => $dir});
+    $bmwqemu::openqa_default_share = $data_dir;
 
     eval {
         use bmwqemu ();
         bmwqemu::init;
+        bmwqemu::ensure_valid_vars();
     };
-    ok($@, 'bmwqemu init failed');
+    ok(!$@, 'init successful');
 
     my %vars = %{read_vars()};
     ok(!$vars{DISTRI}, 'DISTRI not supplied and not set');
-    is($vars{CASEDIR}, $dir, 'CASEDIR unchanged');
-    ok(!$vars{PRJDIR}, 'PRJDIR not supplied and not set');
+    is($vars{CASEDIR}, $dir,      'CASEDIR unchanged');
+    is($vars{PRJDIR},  $data_dir, 'PRJDIR set to default');
 };
 
 done_testing;

diff --git a/t/14-isotovideo.t b/t/14-isotovideo.t
@@ -4,24 +4,47 @@ use strict;
 use warnings;
 use Test::More;
 use File::Basename;
+use File::Path;
 use Cwd 'abs_path';
 
 my $toplevel_dir = abs_path(dirname(__FILE__) . '/..');
-my $data_dir     = "$toplevel_dir/t/data/";
-my $pool_dir     = "$toplevel_dir/t/pool/";
+my $data_dir     = "$toplevel_dir/t/data";
+my $pool_dir     = "$toplevel_dir/t/pool";
 
-chdir($pool_dir);
-open(my $var, '>', 'vars.json');
-print $var <<EOV;
+sub isotovideo {
+    my (%args) = @_;
+    $args{opts} //= '';
+    system("perl $toplevel_dir/isotovideo -d $args{opts} 2>&1 | tee autoinst-log.txt");
+    is(system('grep -q "\d*: EXIT 1" autoinst-log.txt'), 0, $args{end_test_str} ? $args{end_test_str} : 'isotovideo run exited as expected');
+}
+
+sub is_in_log {
+    my ($regex, $msg) = @_;
+    is(system("grep -q \"$regex\" autoinst-log.txt"), 0, $msg);
+}
+
+subtest 'standalone isotovideo without vars.json file and only command line parameters' => sub {
+    chdir($pool_dir);
+    unlink('vars.json');
+    isotovideo(opts => "casedir=$data_dir/tests schedule=foo,bar/baz _exit_after_schedule=1");
+    is_in_log('scheduling.*foo',     'requested modules are run as part of enforced scheduled');
+    is_in_log('scheduling.*bar/baz', 'requested modules in subdirs are scheduled');
+};
+
+subtest 'standard tests based on simple vars.json file' => sub {
+    chdir($pool_dir);
+    open(my $var, '>', 'vars.json');
+    print $var <<EOV;
 {
    "CASEDIR" : "$data_dir/tests",
    "PRJDIR"  : "$data_dir",
    "_EXIT_AFTER_SCHEDULE" : 1,
 }
 EOV
-close($var);
-system("perl $toplevel_dir/isotovideo -d 2>&1 | tee autoinst-log.txt");
-is(system('grep -q "\d*: EXIT 1" autoinst-log.txt'),              0, 'test exited early as requested');
-is(system('grep -q "\d* scheduling.*shutdown" autoinst-log.txt'), 0, 'schedule has been evaluated');
+    close($var);
+    isotovideo;
+    is_in_log('\d*: EXIT 1',              'test exited early as requested');
+    is_in_log('\d* scheduling.*shutdown', 'schedule has been evaluated');
+};
 
 done_testing();
diff --git a/t/data/tests/tests/bar/baz.pm b/t/data/tests/tests/bar/baz.pm
@@ -0,0 +1,3 @@
+use strict;
+use base 'basetest';
+1;
diff --git a/t/data/tests/tests/foo.pm b/t/data/tests/tests/foo.pm
@@ -0,0 +1,3 @@
+use strict;
+use base 'basetest';
+1;