JeOS: expand image information

os-autoinst · May 10, 2022 · 3d92313 · 3d92313
1 parent ecbd24f
commit 3d92313
Show file tree

Hide file tree

Showing 4 changed files with 125 additions and 47 deletions.
diff --git a/lib/main_common.pm b/lib/main_common.pm
@@ -574,7 +574,6 @@ sub load_system_role_tests {
 sub load_jeos_openstack_tests {
     return unless is_openstack;
     my $args = OpenQA::Test::RunArgs->new();
-    loadtest "jeos/image_info";
     loadtest 'boot/boot_to_desktop';
     if (get_var('JEOS_OPENSTACK_UPLOAD_IMG')) {
         loadtest "publiccloud/upload_image";
@@ -589,7 +588,7 @@ sub load_jeos_openstack_tests {
     } else {
         loadtest 'publiccloud/ssh_interactive_start', run_args => $args;
     }
-
+    loadtest "jeos/image_info";
     loadtest "jeos/record_machine_id";
     loadtest "console/system_prepare" if is_sle;
     loadtest "console/force_scheduled_tasks";
@@ -615,8 +614,8 @@ sub load_jeos_tests {
         loadtest "jeos/prepare_firstboot";
     }
     load_boot_tests();
-    loadtest "jeos/image_info";
     loadtest "jeos/firstrun";
+    loadtest "jeos/image_info";
     loadtest "jeos/record_machine_id";
     loadtest "console/system_prepare" if is_sle;
     loadtest "console/force_scheduled_tasks";

diff --git a/schedule/jeos/sle/jeos-main.yaml b/schedule/jeos/sle/jeos-main.yaml
@@ -31,8 +31,8 @@ conditional_schedule:
                 - qa_automation/patch_and_reboot
 schedule:
     - '{{bootloader}}'
-    - jeos/image_info
     - jeos/firstrun
+    - jeos/image_info
     - jeos/record_machine_id
     - console/system_prepare
     - console/force_scheduled_tasks

diff --git a/schedule/jeos/sle/rpi/jeos-main.yaml b/schedule/jeos/sle/rpi/jeos-main.yaml
@@ -2,8 +2,8 @@
 description: 'Main RPi test suite. Maintainer: dheidler.'
 name: 'jeos-main@aarch64 ( qemu )'
 schedule:
-  - jeos/image_info
   - jeos/firstrun
+  - jeos/image_info
   - jeos/record_machine_id
   - console/system_prepare
   - console/force_scheduled_tasks

diff --git a/tests/jeos/image_info.pm b/tests/jeos/image_info.pm
@@ -19,22 +19,44 @@ use version_utils qw(is_sle is_opensuse is_openstack);
 use mmapi qw(get_current_job_id);
 use Cwd;
 
-sub ls {
-    my $output = `ls -l`;
-    record_info('ls', $output);
+our $image;
+our $job_url;
+our $db_log = "/tmp/influxdb.txt";
+
+sub push_to_influxdb {
+    my ($self, %args) = @_;
+    $args{influxdb_server} //= get_required_var('INFLUXDB_SERVER');
+
+    my $query = 'curl -K influxdb_conf -i -XPOST "http://' . $args{influxdb_server} . ':8086/write?db=data"';
+    $query .= ' --data-binary "' . $args{table};
+    $query .= ',distri=' . get_required_var('DISTRI');
+    $query .= ',version=' . get_required_var('VERSION');
+    $query .= ',arch=' . get_required_var('ARCH');
+    $query .= ',flavor=' . get_required_var('FLAVOR');
+    $query .= ',build=' . get_required_var('BUILD');
+    $query .= ',image=' . $image;
+    $query .= ',job_url=' . $job_url;
+    if ($args{extra}) {
+        foreach (keys %{$args{extra}}) {
+            $query .= ',' . $_ . '=' . $args{extra}->{$_};
+        }
+    }
+    $query .= ' value=' . $args{value} . '"';
+    script_run("echo '$query' | tee -a $db_log");
+    my $query_output = script_output("$query 2>&1 | tee -a $db_log", proceed_on_failure => 1);
+    # if successful push, it should return 'HTTP/1.1 204 No Content'
+    record_info('DB failure', "There has been a problem pushing data to InfluxDB.") unless ($query_output =~ /(?=.*204 No Content)/);
 }
 
 sub run {
     my $self = shift;
-    my $download_url;
-    my $hdd;
-    my $image;
-    my $size;
+    set_var('_QUIET_SCRIPT_CALLS', 1);    # Only show record_info frames.
+    $self->select_serial_terminal;
 
+    my $hdd;
+    my $image_size;
     my $dir = getcwd;
-    record_info('pwd', $dir);    # For debug purposes
 
-    ls;    # debug
     if (is_openstack) {
         # In OpenStack JeOS tests, we boot a different HDD which serves as
         # a jumphost with the needed CLI tools to upload the image and create
@@ -50,79 +72,136 @@ sub run {
     }
 
     record_info('HDD', $hdd);
-    ls;
 
     if ($hdd =~ /\.xz/) {
         # We want to monitor the size of uncompressed images.
         my $cmd = "nice ionice unxz -k $hdd -c > hdd_uncompressed";
         record_info('unxz', "Extracting compressed file to get its size.\n$cmd");
         system($cmd);
-        $size = -s 'hdd_uncompressed';
-        ls;    # debug
+        $image_size = -s 'hdd_uncompressed';
         system("rm hdd_uncompressed");
         ($image = $hdd) =~ s/\.xz//;
     } else {
         $image = $hdd;
-        $size = -s $hdd;
+        $image_size = -s $hdd;
     }
-    my $size_mb = $size / 1024 / 1024;
+    my $size_mb = $image_size / 1024 / 1024;
     record_info('Image', "Image: $image\nSize: $size_mb");
 
-    # Record this value in the InfluxDB
+    # Get list of packages installed in the system
+    my $packages = script_output('rpm -qa --queryformat "%{SIZE} %{NAME}\n" |sort -n -r');
+    my @rpm_array = split(/\n/, $packages);
+    my $num_packages = scalar @rpm_array;
+    record_info('rpm total', "Total number of installed packages: $num_packages");
+    record_info('rpm list', $packages);
+
+    # Get the size of different file types
+    my $btrfs_summary = script_output('btrfs filesystem df --mbytes --si /');
+    record_info('btrfs', "$btrfs_summary");
+    my ($data) = $btrfs_summary =~ /Data.*/g;
+    my ($data_mb) = $data =~ /(\d+\.\d+)/g;
+    my ($system) = $btrfs_summary =~ /System.*/g;
+    my ($system_mb) = $system =~ /(\d+\.\d+)/g;
+    my ($metadata) = $btrfs_summary =~ /Metadata.*/g;
+    my ($metadata_mb) = $metadata =~ /(\d+\.\d+)/g;
+    my ($globalreserve) = $btrfs_summary =~ /GlobalReserve.*/g;
+    my ($globalreserve_mb) = $globalreserve =~ /(\d+\.\d+)/g;
+    record_info('Data', "$data_mb MB");
+    record_info('System', "$system_mb MB");
+    record_info('Metadata', "$metadata_mb MB");
+    record_info('GlobalReserve', "$globalreserve_mb MB");
+
+    # Get size of the root directories in KB (except /proc).
+    my $dirs = script_output('du -d 1 --block-size=1K --exclude=/proc /');
+    record_info("dirs", $dirs);
+
+    # Record all this information in the InfluxDB
     # We only allow this on certain conditions:
+    #  - The job must contain INFLUXDB_SERVER, _SECRET_INFLUXDB_USER and _SECRET_INFLUXDB_PWD
+    #  - The job shouldn't be a verification run
     #  - The job must be executed from OSD or O3
-    #  - The job must contain INFLUXDB_SERVER and _SECRET_INFLUXDB_USER and _SECRET_INFLUXDB_PWD
-    #  - The job shouldn't be a verification run => CASEDIR variable
     my $influxdb_server = get_var('INFLUXDB_SERVER');
     my $influxdb_user = get_var('_SECRET_INFLUXDB_USER');
     my $influxdb_pwd = get_var('_SECRET_INFLUXDB_PWD');
+    return unless ($influxdb_server);
+    unless ($influxdb_user || $influxdb_pwd) {
+        record_info('INFO', 'INFLUXDB_SERVER variable is set but "_SECRET_INFLUXDB_USER" and "_SECRET_INFLUXDB_PWD" are missing.');
+        return;
+    }
+    # To not let openQA show the echo command that exposes user/password on the UI, we need to make
+    # a local perl call and store it to a file in the worker and upload it to the VM
+    system("echo '-u $influxdb_user:$influxdb_pwd' > $dir/os-autoinst-distri-opensuse/data/influxdb_conf");
+    assert_script_run('curl -s ' . data_url('influxdb_conf') . ' -o /root/influxdb_conf');
 
-    return if (get_var('CASEDIR') !~ m/^sle$|^opensuse$/);
-    return unless ($influxdb_server || $influxdb_user || $influxdb_pwd);
+    # CASEDIR var is always set, and if when no specified, the value is "sle" for OSD and "opensuse" for O3
+    #return if (get_var('CASEDIR') !~ m/^sle$|^opensuse$/);
 
-    my $job_url;
     my $openqa_host = get_required_var('OPENQA_HOSTNAME');
     if ($openqa_host =~ /openqa1-opensuse|openqa.opensuse.org/) {    # O3 hostname
         $job_url = 'https://openqa.opensuse.org/tests/' . get_current_job_id();
     }
     elsif ($openqa_host =~ /openqa.suse.de/) {    # OSD hostname
         $job_url = 'https://openqa.suse.de/tests/' . get_current_job_id();
     }
-
     return unless ($job_url);
 
     # Check if the data for this image has been already published before.
     # This will avoid publishing the same data if a job is restarted.
-    system("echo '-u $influxdb_user:$influxdb_pwd' > $dir/influxdb_conf");
-
-    my $query = 'curl -K ' . $dir . '/influxdb_conf -i -G "http://' . $influxdb_server . ':8086/query?db=data"';
+    my $query = 'curl -K influxdb_conf -i -G "http://' . $influxdb_server . ':8086/query?db=data"';
     $query .= ' --data-urlencode "q=SELECT \"value\" FROM \"image_size\" WHERE \"image\" = \'' . $image . '\'"';
-    my $query_output = `$query 2>&1`;
+    script_run("echo '$query' | tee -a $db_log");
+    my $query_output = script_output("$query 2>&1 | tee -a $db_log", proceed_on_failure => 1);
     record_info('DB check', "Check if this value exists in the DB.\nquery = $query\nresult = \n$query_output");
     # Successful query should return 'HTTP/1.1 200 OK'
     # Empty records will return  '{"results":[{"statement_id":0}]}'
     # Existing records will return '{"results":[{"statement_id":0,"series":[{"name":"size","columns":["time","value"],"values"' ...
     if ($query_output !~ /200 OK/) {
-        record_soft_failure("poo#110221 - There has been a problem with the query.");
+        record_info('DB Failure', 'There has been a problem with the query.');
+        return;
     } elsif ($query_output =~ /values/) {
         record_info('Skip', "Skip pushing the data to the DB. There is already a DB record for this image.");
-    } else {
-        $query = 'curl -K influxdb_conf -i -XPOST "http://' . $influxdb_server . ':8086/write?db=data"';
-        $query .= ' --data-binary "image_size';
-        $query .= ',distri=' . get_required_var('DISTRI');
-        $query .= ',version=' . get_required_var('VERSION');
-        $query .= ',arch=' . get_required_var('ARCH');
-        $query .= ',flavor=' . get_required_var('FLAVOR');
-        $query .= ',build=' . get_required_var('BUILD');
-        $query .= ',image=' . $image;
-        $query .= ',job_url=' . $job_url;
-        $query .= ',type=uncompressed';
-        $query .= ' value=' . $size_mb . '"';
-        my $query_output = `$query 2>&1`;
-        record_info('DB push', "query = $query\nresult = \n$query_output");
-        # if successful push, it should return 'HTTP/1.1 204 No Content'
-        record_soft_failure("poo#110221 - There has been a problem pushing data to InfluxDB.") unless ($query_output =~ /(?=.*204 No Content)/);
+        return;
     }
+
+    # Push image size
+    $self->push_to_influxdb(table => 'image_size', value => $size_mb, extra => {type => 'uncompressed'});
+
+    # Push number of packages
+    $self->push_to_influxdb(table => 'num_packages', value => $num_packages);
+
+    # Push each rpm size
+    my @lines = split /\n/, $packages;
+    foreach my $line (@lines) {
+        my ($size) = $line =~ /^\d+/g;
+        my ($rpm) = $line =~ /\s(.*)/g;
+        $self->push_to_influxdb(table => 'rpms', value => $size, extra => {rpm => $rpm});
+    }
+
+    # Push btrfs sizes
+    $self->push_to_influxdb(table => 'btrfs_df', value => $data_mb, extra => {type => 'Data'});
+    $self->push_to_influxdb(table => 'btrfs_df', value => $system_mb, extra => {type => 'System'});
+    $self->push_to_influxdb(table => 'btrfs_df', value => $metadata_mb, extra => {type => 'Metadata'});
+    $self->push_to_influxdb(table => 'btrfs_df', value => $globalreserve_mb, extra => {type => 'GlobalReserve'});
+
+    # Push dir sizes
+    @lines = split /\n/, $dirs;
+    foreach my $line (@lines) {
+        my ($size) = $line =~ /^\d+/g;
+        my ($dir) = $line =~ /\s(.*)/g;
+        $self->push_to_influxdb(table => 'directories', value => $size, extra => {dir => $dir});
+    }
+    record_info('DB', 'The data has been pushed to the DataBase.');
+}
+
+
+sub post_run_hook {
+    upload_logs($db_log, failok => 1);
+    set_var('_QUIET_SCRIPT_CALLS', 0);
+}
+
+sub post_fail_hook {
+    upload_logs($db_log, failok => 1);
+    set_var('_QUIET_SCRIPT_CALLS', 0);
 }
 
 1;