Skip to content

Commit

Permalink
Rewrite code for job settings expansion
Browse files Browse the repository at this point in the history
* Move expand placeholders function to its own module.
* rewrite the loop.

See: https://progress.opensuse.org/issues/20464
  • Loading branch information
Amrysliu committed Jun 10, 2019
1 parent 88bdc79 commit 8666a6b
Show file tree
Hide file tree
Showing 6 changed files with 292 additions and 55 deletions.
20 changes: 12 additions & 8 deletions lib/OpenQA/Schema/Result/ScheduledProducts.pm
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ use DBIx::Class::Timestamps 'now';
use File::Basename;
use Try::Tiny;
use OpenQA::Utils;
use OpenQA::Settings;
use OpenQA::JobDependencies::Constants;
use Mojo::JSON qw(encode_json decode_json);
use Carp;
Expand Down Expand Up @@ -218,7 +219,9 @@ sub _schedule_iso {
# Any arg name ending in _URL is special: it tells us to download
# the file at that URL before running the job
my $downloads = create_downloads_list($args);
my $jobs = $self->_generate_jobs($args, \@notes);
my $result = $self->_generate_jobs($args, \@notes);
return {error => $result->{expand_result}} if defined $result->{expand_result};
my $jobs = $result->{settings_result};

# take some attributes from the first job to guess what old jobs to cancel
# note: We should have distri object that decides which attributes are relevant here.
Expand Down Expand Up @@ -507,6 +510,7 @@ sub _generate_jobs {
# allow overriding the priority
my $priority = delete $args->{_PRIORITY};

my $expand_result;
for my $product (@products) {
# find job templates
my $templates = $product->job_templates;
Expand Down Expand Up @@ -565,23 +569,23 @@ sub _generate_jobs {

# variable expansion
# replace %NAME% with $settings{NAME}
my $new_settings = OpenQA::Utils::reset_settings(\%settings);
$expand_result = OpenQA::Settings::expand_placeholders(\%settings);

if (!$args->{MACHINE} || $args->{MACHINE} eq $new_settings->{MACHINE}) {
if (!$args->{MACHINE} || $args->{MACHINE} eq $settings{MACHINE}) {
if (!@tests) {
$wanted{_settings_key($new_settings)} = 1;
$wanted{_settings_key(\%settings)} = 1;
}
else {
foreach my $test (@tests) {
if ($test eq $new_settings->{TEST}) {
$wanted{_settings_key($new_settings)} = 1;
if ($test eq $settings{TEST}) {
$wanted{_settings_key(\%settings)} = 1;
last;
}
}
}
}

push @$ret, $new_settings;
push @$ret, \%settings;
}
}

Expand All @@ -602,7 +606,7 @@ sub _generate_jobs {
splice @$ret, $i, 1; # not wanted - delete
}
}
return $ret;
return {expand_result => $expand_result, settings_result => $ret};
}

=over 4
Expand Down
55 changes: 55 additions & 0 deletions lib/OpenQA/Settings.pm
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
# Copyright (C) 2019 SUSE LLC
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License

package OpenQA::Settings;

use strict;
use warnings;

# replace %NAME% with $settings{NAME}
sub expand_placeholders {
my ($settings) = @_;

my $error_string;

for my $value (values %$settings) {
next unless defined $value;

# loop the value until there is no %KEY% in it
while ($value =~ /%(\w+)%/) {
my $key = $1;
last unless defined $settings->{$key};

my $replaced_value = $settings->{$key};

# used to record already replaced value. If the value has been in this array which means there is a circular reference, then return an error message.
my @replaced_array = ();

while ($replaced_value =~ /%(\w+)%/) {
last unless defined $settings->{$1};
if (grep { $_ eq $1 } @replaced_array) {
return "The key $1 is a circular reference, value is " . $settings->{$1};
}
else {
$replaced_value = $settings->{$1};
push @replaced_array, $1;
}
}
$value =~ s/(%$key%)/$replaced_value/;
}
}
return undef;
}

1;
25 changes: 0 additions & 25 deletions lib/OpenQA/Utils.pm
Original file line number Diff line number Diff line change
Expand Up @@ -1241,30 +1241,5 @@ sub any_array_item_contained_by_hash {
return 0;
}

#replace %NAME% with $settings{NAME}
sub reset_settings {
my ($settings) = @_;

for my $value (values %$settings) {
if (defined $value) {
$value =~ s/(%\w+%)/_replace($settings, $1)/eig;
}
}
return $settings;
}

#only used in reset_settings method
sub _replace {
my ($settings, $param) = @_;

my $key = $param;
$key =~ s/%//g;

if (defined $settings->{$key}) {
return $settings->{$key};
}
return $param;
}

1;
# vim: set sw=4 et:
22 changes: 13 additions & 9 deletions lib/OpenQA/WebAPI/Controller/API/V1/Job.pm
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,12 @@
package OpenQA::WebAPI::Controller::API::V1::Job;
use Mojo::Base 'Mojolicious::Controller';

use OpenQA::Utils;
use OpenQA::Jobs::Constants;
use OpenQA::Resource::Jobs;
use OpenQA::Schema::Result::Jobs;
use OpenQA::Events;
use OpenQA::Utils;
use OpenQA::Settings;
use Try::Tiny;
use DBIx::Class::Timestamps 'now';
use Mojo::Asset::Memory;
Expand Down Expand Up @@ -235,10 +236,12 @@ sub create {

my $json = {};
my $status;
my $new_params = $self->_generate_job_setting(\%params);
my $result = $self->_generate_job_setting(\%params);
return $self->render(json => {error => $result->{expand_result}}, status => 400)
if defined $result->{expand_result};

try {
my $job = $self->schema->resultset('Jobs')->create_from_settings($new_params);
my $job = $self->schema->resultset('Jobs')->create_from_settings($result->{settings_result});
$self->emit_event('openqa_job_create', {id => $job->id, %params});
$json->{id} = $job->id;

Expand Down Expand Up @@ -754,10 +757,10 @@ sub _generate_job_setting {
my ($self, $args) = @_;
my $schema = $self->schema;

my %settings; #include the machine, product, and test suit setting belongs to this job.
my @classes; #if the machine or product has worker_class setting, push them to this array.
my %settings; # Machines, product and test suite settings for the job
my @classes; # Populated with WORKER_CLASS settings from machines and products

#if args includes DISTRI, VERSION, FLAVOR, ARCH, get product setting.
# Populated with Product settins if there are DISTRI, VERSION, FLAVOR, ARCH in arguments.
if ( defined $args->{DISTRI}
&& defined $args->{VERSION}
&& defined $args->{FLAVOR}
Expand All @@ -781,7 +784,7 @@ sub _generate_job_setting {
}
}

#if args includes MACHINE, get machine setting.
# Populated with machine settings if there is MACHINE in arguments.
if (defined $args->{MACHINE}) {
my $machines = $schema->resultset('Machines')->search(
{
Expand All @@ -798,7 +801,7 @@ sub _generate_job_setting {
}
}

#TEST is mandatory, find the test suit settings.
# TEST is mandatory, so populate with TestSuit settings.
my $test_suites = $schema->resultset('TestSuites')->search(
{
name => $args->{TEST},
Expand All @@ -821,7 +824,8 @@ sub _generate_job_setting {
$settings{uc $_} = $args->{$_};
}

return OpenQA::Utils::reset_settings(\%settings);
my $expand_result = OpenQA::Settings::expand_placeholders(\%settings);
return {expand_result => $expand_result, settings_result => \%settings};
}

1;
Expand Down
120 changes: 120 additions & 0 deletions t/40-settings.t
Original file line number Diff line number Diff line change
@@ -0,0 +1,120 @@
#! /usr/bin/perl

# Copyright (C) 2019 SUSE LLC
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

use strict;
use warnings;

BEGIN {
unshift @INC, 'lib';
}

use FindBin;
use lib "$FindBin::Bin/lib";
use Test::More;
use OpenQA::Settings;

my $settings = {
BUILD_SDK => '%BUILD_HA%',
BETA => 1,
ISO_MAXSIZE => '4700372992',
SHUTDOWN_NEEDS_AUTH => 1,
HDD_1 => 'SLES-%VERSION%-%ARCH%-%BUILD%@%MACHINE%-minimal_with_sdk%BUILD_SDK%_installed.qcow2',
PUBLISH_HDD_1 => 'SLES-%VERSION%-%ARCH%-%BUILD%@%MACHINE%-minimal_with_sdk%BUILD_SDK%_installed.qcow2',
ANOTHER_JOB => 'SLES-%VERSION%-%ARCH%-%BUILD%@%MACHINE%-minimal_with_sdk%BUILD_SDK%_installed.qcow2',
ARCH => 'x86_64',
BACKEND => 'qemu',
BUILD => '1234',
BUILD_SLE => '%BUILD%',
MACHINE => '64bit',
PATCH => 1,
UPGRADE => 1,
ISO => 'SLE-%VERSION%-%FLAVOR%-%MACHINE%-Build%BUILD%-Media1.iso',
WORKER_CLASS => 'qemu_x86_64',
VERSION => '15-SP1',
FLAVOR => 'Installer-DVD',
ADDONURL_SDK => 'ftp://openqa.suse.de/SLE-%VERSION%-SDK-POOL-%ARCH%-Build%BUILD_SDK%-Media1/',
DEPENDENCY_RESOLVER_FLAG => 1,
DESKTOP => 'textmode',
DEV_IMAGE => 1,
HDDSIZEGB => 50,
INSTALLONLY => 1,
PATTERNS => 'base,minimal',
PUBLISH_PFLASH_VARS =>
'SLES-%VERSION%-%ARCH%-%BUILD%@%MACHINE%-minimal_with_sdk%BUILD_SDK%_installed-uefi-vars.qcow2',
SEPARATE_HOME => 0,
BUILD_HA => '%BUILD%',
BUILD_SES => '%BUILD%',
WORKAROUND_MODULES => 'base,desktop,serverapp,script,sdk',
};

subtest expand_placeholders => sub {
my $str = OpenQA::Settings::expand_placeholders($settings);
my $match_settings = {
BUILD_SDK => '1234',
BETA => 1,
ISO_MAXSIZE => '4700372992',
SHUTDOWN_NEEDS_AUTH => 1,
HDD_1 => 'SLES-15-SP1-x86_64-1234@64bit-minimal_with_sdk1234_installed.qcow2',
PUBLISH_HDD_1 => 'SLES-15-SP1-x86_64-1234@64bit-minimal_with_sdk1234_installed.qcow2',
ANOTHER_JOB => 'SLES-15-SP1-x86_64-1234@64bit-minimal_with_sdk1234_installed.qcow2',
ARCH => 'x86_64',
BACKEND => 'qemu',
BUILD => '1234',
BUILD_SLE => '1234',
MACHINE => '64bit',
PATCH => 1,
UPGRADE => 1,
ISO => 'SLE-15-SP1-Installer-DVD-64bit-Build1234-Media1.iso',
WORKER_CLASS => 'qemu_x86_64',
VERSION => '15-SP1',
FLAVOR => 'Installer-DVD',
ADDONURL_SDK => 'ftp://openqa.suse.de/SLE-15-SP1-SDK-POOL-x86_64-Build1234-Media1/',
DEPENDENCY_RESOLVER_FLAG => 1,
DESKTOP => 'textmode',
DEV_IMAGE => 1,
HDDSIZEGB => 50,
INSTALLONLY => 1,
PATTERNS => 'base,minimal',
PUBLISH_PFLASH_VARS => 'SLES-15-SP1-x86_64-1234@64bit-minimal_with_sdk1234_installed-uefi-vars.qcow2',
SEPARATE_HOME => 0,
BUILD_HA => '1234',
BUILD_SES => '1234',
WORKAROUND_MODULES => 'base,desktop,serverapp,script,sdk',
};
is_deeply($settings, $match_settings, "Settings replaced");
};

subtest circular_reference => sub {
my $circular_settings = {
BUILD_SDK => '%BUILD_HA%',
ISO_MAXSIZE => '4700372992',
HDD_1 => 'SLES-%VERSION%-%ARCH%-%BUILD_HA%@%MACHINE%-minimal_with_sdk%BUILD_SDK%_installed.qcow2',
PUBLISH_HDD_1 => 'SLES-%VERSION%-%ARCH%-%BUILD_HA%@%MACHINE%-minimal_with_sdk%BUILD_SDK%_installed.qcow2',
ANOTHER_JOB => 'SLES-%VERSION%-%ARCH%-%BUILD_HA%@%MACHINE%-minimal_with_sdk%BUILD_SDK%_installed.qcow2',
ARCH => 'x86_64',
BACKEND => 'qemu',
BUILD => '%BUILD_HA%',
BUILD_HA => '%BUILD%',
VERSION => '15-SP1',
MACHINE => '64bit',
};
my $str = OpenQA::Settings::expand_placeholders($circular_settings);
like($str, qr/The key (\w+) is a circular reference, value is %\w+%/, "circular reference exit successfully");
};

done_testing;
Loading

0 comments on commit 8666a6b

Please sign in to comment.