qa_automation: add new design for QA:HEAD tests

-Split test workflow into 4 steps;
-Define functions to converting ctcs2 log to junit xml;
-Define functions to upload system logs;

lib/ctcs2_to_junit.pm

@@ -0,0 +1,117 @@
+# SUSE's openQA tests
+#
+# Copyright © 2017 SUSE LLC
+#
+# Copying and distribution of this file, with or without modification,
+# are permitted in any medium without royalty provided the copyright
+# notice and this notice are preserved.  This file is offered as-is,
+# without any warranty.
+#
+# Summary:  base class for convert QA:Head ctcs2 wrapper testsuite test log to junit format
+# Maintainer: Yong Sun <yosun@suse.com>
+
+package ctcs2_to_junit;
+
+use strict;
+use warnings;
+use base "Exporter";
+use Exporter;
+use testapi;
+use utils;
+use XML::Writer;
+
+our @EXPORT = qw(analyzeResult generateXML);
+
+sub analyzeResult {
+    my ($text) = @_;
+    my $result;
+    $text =~ /Test in progress(.*)Test run complete/s;
+    my $rough_result = $1;
+    foreach (split("\n", $rough_result)) {
+        if ($_ =~ /(\S+)\s+\.{3}\s+\.{3}\s+(PASSED|FAILED|SKIPPED)\s+\((\S+)\)/g) {
+            $result->{$1}{status} = $2;
+            $result->{$1}{time}   = $3;
+        }
+    }
+    return $result;
+}
+
+sub generateXML {
+    my ($data) = @_;
+
+    my %my_hash = %$data;
+    my $case_status;
+    my $case_num = scalar(keys %my_hash);
+    my $pass_num = 0;
+    my $fail_num = 0;
+    my $skip_num = 0;
+    my $writer   = new XML::Writer(DATA_MODE => 'true', DATA_INDENT => 2, OUTPUT => "self");
+
+    foreach my $item (keys(%my_hash)) {
+        if ($my_hash{$item}->{status} =~ m/PASSED/) {
+            $pass_num += 1;
+        }
+        elsif ($my_hash{$item}->{status} =~ m/SKIPPED/) {
+            $skip_num += 1;
+        }
+        else {
+            $fail_num += 1;
+        }
+    }
+    $writer->startTag(
+        'testsuites',
+        error    => "0",
+        failures => "$fail_num",
+        name     => "",
+        skipped  => "$skip_num",
+        tests    => "$case_num",
+        time     => ""
+    );
+    $writer->startTag(
+        'testsuite',
+        error     => "0",
+        failures  => "$fail_num",
+        hostname  => `hostname`,
+        id        => "0",
+        name      => get_var("QA_TESTSUITE"),
+        package   => get_var("QA_TESTSUITE"),
+        skipped   => "0",
+        tests     => "$case_num",
+        time      => "",
+        timestamp => `date`
+    );
+
+    foreach my $item (keys(%my_hash)) {
+        if ($my_hash{$item}->{status} =~ m/PASSED/) {
+            $case_status = "success";
+        }
+        elsif ($my_hash{$item}->{status} =~ m/SKIPPED/) {
+            $case_status = "skipped";
+        }
+        else {
+            $case_status = "failure";
+        }
+
+        $writer->startTag(
+            'testcase',
+            classname => get_var("QA_TESTSUITE"),
+            name      => $item,
+            status    => $case_status,
+            time      => $my_hash{$item}->{time});
+        $writer->startTag('system-err');
+        $writer->characters("");
+        $writer->endTag('system-err');
+        $writer->startTag('system-out');
+        $writer->characters("");
+        $writer->endTag('system-out');
+        $writer->endTag('testcase');
+    }
+
+    $writer->endTag('testsuite');
+    $writer->endTag('testsuites');
+
+    $writer->end();
+    $writer->to_string();
+}
+
+1;

lib/upload_system_log.pm

@@ -0,0 +1,62 @@
+# SUSE's openQA tests
+#
+#Copyright © 2017 SUSE LLC
+#
+# Copying and distribution of this file, with or without modification,
+# are permitted in any medium without royalty provided the copyright
+# notice and this notice are preserved.  This file is offered as-is,
+# without any warranty.
+#
+package upload_system_log;
+# Summary:  base class for collecting and uploading system log
+# Maintainer: Yong Sun <yosun@suse.com>
+
+use strict;
+use base "Exporter";
+use Exporter;
+use testapi;
+use utils;
+use base "opensusebasetest";
+
+our @EXPORT = qw(upload_system_logs);
+
+# Save the output of $cmd into $file and upload it
+sub system_status {
+    my ($self, $log) = @_;
+    $log //= "/tmp/system-status.log";
+
+    my @klst = ("kernel", "cpuinfo", "memory", "repos", "lspci");
+    my %cmds = (
+        kernel  => "uname -a",
+        cpuinfo => "cat /proc/cpuinfo",
+        memory  => "free -m",
+        repos   => "zypper repos -u",
+        lspci   => "lspci",
+    );
+
+    foreach my $key (@klst) {
+        my $cmd = "echo '=========> $key <=========' >> $log; ";
+        $cmd .= "$cmds{$key} >> $log; ";
+        $cmd .= "echo '' >> $log";
+        script_run($cmd, 40);
+    }
+    return $log;
+}
+
+sub journalctl_log {
+    my ($self, $sys_log) = @_;
+    $sys_log //= "/tmp/journalctl.log";
+    script_run("journalctl -b >$sys_log", 40);
+
+    return $sys_log;
+}
+
+sub upload_system_logs {
+    my $log     = system_status();
+    my $sys_log = journalctl_log();
+
+    upload_logs($log,     timeout => 100);
+    upload_logs($sys_log, timeout => 100);
+}
+
+1;

products/sle/main.pm

@@ -1252,6 +1252,12 @@ elsif (get_var("QA_TESTSET")) {
     }
     loadtest "qa_automation/" . get_var("QA_TESTSET");
 }
+elsif (get_var("QA_TESTSUITE")) {
+    boot_hdd_image;
+    loadtest "qa_automation/prepare_qa_repo";
+    loadtest "qa_automation/install_test_suite";
+    loadtest "qa_automation/execute_test_run";
+}
 elsif (get_var("XFSTESTS")) {
     loadtest "qa_automation/xfstests_prepare_boot";
     loadtest "qa_automation/xfstests_prepare_testsuite";

tests/qa_automation/execute_test_run.pm

@@ -0,0 +1,55 @@
+# SUSE's openQA tests
+#
+# Copyright © 2017 SUSE LLC
+#
+# Copying and distribution of this file, with or without modification,
+# are permitted in any medium without royalty provided the copyright
+# notice and this notice are preserved.  This file is offered as-is,
+# without any warranty.
+#
+# Summary:  to execute testsuites in openQA from QA:Head by running its build up test run script
+# Maintainer: Yong Sun  <yosun@suse.com>
+
+use strict;
+use File::Basename;
+use IO::File;
+use Data::Dumper;
+use utils;
+use testapi;
+use ctcs2_to_junit;
+use upload_system_log;
+use base "opensusebasetest";
+
+sub run {
+    my $timeout = get_var("MAX_JOB_TIME") || 7200;
+    my $test    = get_var("QA_TESTSUITE") . get_var("QA_VERSION");
+    my $run_log = "/tmp/$test-run.log";
+
+    #execute test run
+    script_run("/usr/share/qa/tools/test_$test-run |tee $run_log", $timeout);
+
+    save_screenshot;
+
+    #output result to serial0 and upload test log
+    if (get_var("QA_TESTSUITE")) {
+        my $test_log = script_output("ls /var/log/qa/ctcs2/");
+        my $tarball  = "/tmp/$test_log.tar.bz2";
+        assert_script_run("tar cjf $tarball -C /var/log/qa/ctcs2 $test_log");
+        upload_logs($tarball, timeout => 600);
+
+        #convert to junit log
+        my $script_output = script_output("cat $run_log");
+        my $tc_result     = analyzeResult($script_output);
+        my $xml_result    = generateXML($tc_result);
+        script_output "echo \'$xml_result\' > /tmp/output.xml", 7200;
+        parse_junit_log("/tmp/output.xml");
+    }
+
+    #upload system log
+    upload_system_logs();
+
+    #assert test result
+    assert_script_run("grep 'Test run completed successfully' $run_log");
+}
+
+1;

tests/qa_automation/install_test_suite.pm

@@ -0,0 +1,27 @@
+# SUSE's openQA tests
+#
+# Copyright © 2017 SUSE LLC
+#
+# Copying and distribution of this file, with or without modification,
+# are permitted in any medium without royalty provided the copyright
+# notice and this notice are preserved.  This file is offered as-is,
+# without any warranty.
+#
+# Summary:  the step to install testsuite from QA:Head
+# Maintainer: Yong Sun <yosun@suse.com>
+
+use strict;
+use base "opensusebasetest";
+use utils;
+use testapi;
+
+sub run {
+    my $test = get_required_var('QA_TESTSUITE');
+    zypper_call("in 'qa_test_$test'");
+}
+
+sub test_flags {
+    return {milestone => 1, fatal => 1};
+}
+
+1;

tests/qa_automation/prepare_qa_repo.pm

@@ -0,0 +1,43 @@
+# SUSE's openQA tests
+#
+# Copyright © 2017 SUSE LLC
+#
+# Copying and distribution of this file, with or without modification,
+# are permitted in any medium without royalty provided the copyright
+# notice and this notice are preserved.  This file is offered as-is,
+# without any warranty.
+#
+# Summary: the step to prepare QA:Head repository
+# Maintainer: Yong Sun <yosun@suse.com>
+
+use strict;
+use testapi;
+use utils;
+use base "opensusebasetest";
+
+# Add qa head repo for kernel testing. If QA_HEAD_REPO is set,
+# remove all existing zypper repos first
+sub prepare_repos {
+    my $self         = shift;
+    my $qa_head_repo = get_required_var('QA_HEAD_REPO', '');
+    my $qa_web_repo  = get_var('QA_WEB_REPO', '');
+
+    zypper_call("--no-gpg-check ar -f $qa_head_repo qa-ibs");
+    if ($qa_web_repo) {
+        zypper_call("--no-gpg-check ar -f $qa_web_repo qa-web");
+    }
+
+    # sometimes updates.suse.com is busy, so we need to wait for possiblye retries
+    zypper_call("--gpg-auto-import-keys ref");
+}
+
+sub run {
+    select_console 'root-console';
+    prepare_repos();
+}
+
+sub test_flags {
+    return {milestone => 1, fatal => 1};
+}
+
+1;