Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Polish OSB plugin #2333

Merged
merged 4 commits into from
Sep 18, 2019
Merged

Polish OSB plugin #2333

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
1b1b0b4
Don't parse XML with regex in OBS plugin
Martchus Sep 18, 2019
d298d86
Reuse Mojo::UserAgent in OBS plugin
Martchus Sep 18, 2019
d2eb8dc
Polish UI of OBS plugin
Martchus Sep 18, 2019
95e8ad3
Don't show nested folders in OBS plugin
Martchus Sep 18, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 15 additions & 16 deletions lib/OpenQA/WebAPI/Plugin/ObsRsync.pm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ sub register {
my ($c, $project) = @_;
my $url = $c->obs_rsync->project_status_url;
return undef unless $url;
return _is_obs_project_status_dirty($url, $project);
return $self->_is_obs_project_status_dirty($url, $project);
});
$app->helper('obs_rsync.check_and_render_error' => \&_check_and_render_error);

Expand Down Expand Up @@ -79,31 +79,30 @@ sub register {
# try to determine whether project is dirty
# undef means status is unknown
sub _is_obs_project_status_dirty {
my ($url, $project) = @_;
my ($self, $url, $project) = @_;
return undef unless $url;

$url =~ s/%%PROJECT/$project/g;
my $ua = Mojo::UserAgent->new;

my $ua = $self->{ua} ||= Mojo::UserAgent->new;
my $res = $ua->get($url)->result;
return undef unless $res->is_success;

return _parse_obs_response_dirty($res->body);
return _parse_obs_response_dirty($res);
}

sub _parse_obs_response_dirty {
my ($body) = @_;

my $dirty;
return 1 if $body =~ /dirty/g;
while ($body =~ /^(.*repository="images".*)/gm) {
my $line = $1;
if ($line =~ /state="([a-z]+)"/) {
return 1 if $1 ne 'published';
$dirty //= 0;
}
my ($res) = @_;

my $results = $res->dom('result');
return undef unless $results->size;

for my $result ($results->each) {
my $attributes = $result->attr;
return 1 if exists $attributes->{dirty};
next if ($attributes->{repository} // '') ne 'images';
return 1 if ($attributes->{state} // '') ne 'published';
}
return $dirty;
return 0;
}

sub _check_and_render_error {
Expand Down
2 changes: 1 addition & 1 deletion lib/OpenQA/WebAPI/Plugin/ObsRsync/Controller/Folders.pm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ sub run {
return undef if $helper->check_and_render_error($folder, $subfolder);

my $full = Mojo::File->new($helper->home, $folder, $subfolder);
my $files = $full->list({dir => 1})->map('basename')->sort->to_array;
my $files = $full->list->map('basename')->sort->to_array;
$self->render(
'ObsRsync_logfiles',
folder => $folder,
Expand Down
149 changes: 83 additions & 66 deletions lib/OpenQA/WebAPI/Plugin/ObsRsync/templates/ObsRsync_folder.html.ep
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,70 +5,87 @@
% end

% title 'OBS synchronization';
<h2><%= 'OBS synchronization for: ' . $obs_project %></h2>
<div class="container">
OBS synchronization is performed in 3 steps.
</div>
<hr>
<div class="container">
<h6>Step 1. Retrieve lists of required files from predefined OBS locations.</h6>
% if (!@$lst_files) {
No files have been read so far
% } else {
Lists retrieved during last run:
% for my $filename (@$lst_files) {
<%= link_to $filename => url_for('plugin_obs_rsync_download_file', folder => $obs_project, subfolder => ".run_last", filename => $filename) %>
|
% }
%
% }
<br>
% if ($read_files_sh) {
Script used to retrieve files:
<%= link_to $read_files_sh => url_for('plugin_obs_rsync_download_file', folder => $obs_project, subfolder => ".run_last", filename => $read_files_sh) %>
% }
</div>
<hr>
<div class="container">
<h6>Step 2. Generate and execute commands to rsync and rename deliverables from remote OBS location (based on lists from Step 1.)</h6>
% if (!@$rsync_commands) {
No logs found for last run
% } else {
Commands executed during last run:
% }
% for my $filename (@$rsync_commands) {
<%= link_to $filename => url_for('plugin_obs_rsync_download_file', folder => $obs_project, subfolder => ".run_last", filename => $filename) %>
|
% }
<br>
% if (@$rsync_sh) {
Script used to generate commands:
% for my $filename (@$rsync_sh) {
<%= link_to $filename => url_for('plugin_obs_rsync_download_file', folder => $obs_project, subfolder => ".run_last", filename => $filename) %>
|
% }
% }
</div>
<hr>
<div class="container">
<h6>Step 3. Generate and execute openQA REST requests to trigger tests (based on lists from Step 1.)</h6>
% if (!@$openqa_commands) {
No logs found for last run
% } else {
Commands executed during last run:
% }
% for my $filename (@$openqa_commands) {
<%= link_to $filename => url_for('plugin_obs_rsync_download_file', folder => $obs_project, subfolder => ".run_last", filename => $filename) %>
% }
<br>
% if ($openqa_sh) {
Script used to generate commands:
<%= link_to $openqa_sh => url_for('plugin_obs_rsync_download_file', folder => $obs_project, subfolder => ".run_last", filename => $openqa_sh) %>
% }
</div>
<hr>
<div class="row-sm-1">
<%= link_to "all logs" => url_for('plugin_obs_rsync_runs', folder => $obs_project) %>
<h2>OBS synchronization for: <%= $obs_project %></h2>
<h3>Synchronization is performed in 3 steps:</h3>
<ul class="list-group">
<li class="list-group-item flex-column align-items-start">
<h5 class="mb-1">Step 1: Retrieve lists of required files from predefined OBS locations.</h5>
<p class="mb-1">
% if (!@$lst_files) {
No files have been read so far.
% } else {
Lists retrieved during last run:
<ul>
% for my $filename (@$lst_files) {
<li><%= link_to $filename => url_for('plugin_obs_rsync_download_file', folder => $obs_project, subfolder => ".run_last", filename => $filename) %></li>
% }
</ul>
% }
</p>
<p class="mb-1">
% if ($read_files_sh) {
Script used to retrieve files:
%= link_to $read_files_sh => url_for('plugin_obs_rsync_download_file', folder => $obs_project, subfolder => ".run_last", filename => $read_files_sh)
% }
</p>
</li>
<li class="list-group-item flex-column align-items-start">
<h5 class="mb-1">Step 2: Generate and execute commands to rsync and rename deliverables from remote OBS location (based on lists from step 1)</h5>
<p class="mb-1">
% if (!@$rsync_commands) {
No logs found for last run.
% } else {
Commands executed during last run:
<ul>
% for my $filename (@$rsync_commands) {
<li><%= link_to $filename => url_for('plugin_obs_rsync_download_file', folder => $obs_project, subfolder => ".run_last", filename => $filename) %></li>
% }
</ul>
% }
</p>
<p class="mb-1">
% if (@$rsync_sh) {
Scripts used to generate commands:
<ul>
% for my $filename (@$rsync_sh) {
<li><%= link_to $filename => url_for('plugin_obs_rsync_download_file', folder => $obs_project, subfolder => ".run_last", filename => $filename) %></li>
% }
</ul>
% }
</p>
</li>
<li class="list-group-item flex-column align-items-start">
<h5 class="mb-1">Step 3: Generate and execute openQA REST requests to trigger tests (based on lists from step 1)</h5>
<p class="mb-1">
% if (!@$openqa_commands) {
No logs found for last run.
% } else {
Commands executed during last run:
<ul>
% for my $filename (@$openqa_commands) {
<li><%= link_to $filename => url_for('plugin_obs_rsync_download_file', folder => $obs_project, subfolder => ".run_last", filename => $filename) %></li>
% }
</ul>
% }
</p>
<p class="mb-1">
% if ($openqa_sh) {
Script used to retrieve files:
%= link_to $openqa_sh => url_for('plugin_obs_rsync_download_file', folder => $obs_project, subfolder => ".run_last", filename => $openqa_sh)
% }
</p>
</li>
</ul>

<div class="card" style="border: none;">
<div class="card-body">
<div class="btn-toolbar" role="toolbar">
<div class="btn-group mr-2" role="group">
%= link_to 'Show all logs' => url_for('plugin_obs_rsync_runs', folder => $obs_project), class => 'btn btn-primary', role => 'button'
</div>
<div class="btn-group mr-2" role="group">
%= link_to 'Sync now' => url_for('plugin_obs_rsync_queue_run', folder => $obs_project), class => 'btn btn-warning', role => 'button'
</div>
</div>
</div>
</div>
<hr>
<%= link_post 'Sync Now!' => url_for('plugin_obs_rsync_queue_run', folder => $obs_project) %>
2 changes: 1 addition & 1 deletion lib/OpenQA/WebAPI/Plugin/ObsRsync/templates/ObsRsync_gru_index.html.ep
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
<table class="table table-striped">
<thead>
<tr>
<th>Id</th>
<th>ID</th>
<th>Task</th>
<th>Args</th>
<th>Created</th>
Expand Down
10 changes: 8 additions & 2 deletions lib/OpenQA/WebAPI/Plugin/ObsRsync/templates/ObsRsync_index.html.ep
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,14 @@

% title 'OBS synchronization folders';
<h2><%= 'OBS synchronization folders ' %></h2>
<ul>
% for my $folder (@$folders) {
<div><%= link_to $folder => url_for('plugin_obs_rsync_folder', folder => $folder) %></div>
<li><%= link_to $folder => url_for('plugin_obs_rsync_folder', folder => $folder) %></li>
% }
</ul>
<hr>
<%= link_to 'View job queue...' => url_for('plugin_obs_rsync_queue') %>
<div class="btn-toolbar" role="toolbar">
<div class="btn-group mr-2" role="group">
%= link_to 'View job queue' => url_for('plugin_obs_rsync_queue'), class => 'btn btn-primary', role => 'button'
</div>
</div>
9 changes: 5 additions & 4 deletions lib/OpenQA/WebAPI/Plugin/ObsRsync/templates/ObsRsync_logfiles.html.ep
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,10 @@
% end

% title 'OBS synchronization Log';
<h2><%= 'OBS synchronization Log: ' . $folder . '/' . $subfolder %></h2>
<h6><%= $full %></h6>
<h2>OBS synchronization log: <%= $folder %>/<%= $subfolder %></h2>
<h5>Full directory path: <%= $full %></h5>
<ul>
% for my $filename (@$files) {
<%= link_to $filename => url_for('plugin_obs_rsync_download_file', folder => $folder, subfolder => $subfolder, filename => $filename) %>
|
<li><%= link_to $filename => url_for('plugin_obs_rsync_download_file', folder => $folder, subfolder => $subfolder, filename => $filename) %></li>
% }
</ul>
13 changes: 7 additions & 6 deletions lib/OpenQA/WebAPI/Plugin/ObsRsync/templates/ObsRsync_logs.html.ep
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,10 @@
% end

% title 'OBS synchronization Logs';
<h2><%= 'OBS synchronization Logs: ' . $folder %></h2>
<h6><%= $full %></h6>
% for my $subfolder (@$subfolders) {
<%= link_to $subfolder => url_for('plugin_obs_rsync_run', folder => $folder, subfolder => $subfolder) %>
|
% }
<h2>OBS synchronization logs: <%= $folder %></h2>
<h5>Full directory path: <%= $full %></h5>
<ul>
% for my $subfolder (@$subfolders) {
<li><%= link_to $subfolder => url_for('plugin_obs_rsync_run', folder => $folder, subfolder => $subfolder) %></li>
% }
</ul>
60 changes: 28 additions & 32 deletions t/ui/27-plugin_obs_rsync_obs_status.t
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,10 @@ use Mojo::IOLoop::ReadWriteProcess::Session 'session';

OpenQA::Test::Case->new->init_data;

my $response_published = '<resultlist state="c181538ad4f4c1be29e73f85b9237653">
my %fake_response_by_project = (
Proj3 => '
<!-- This project is published. -->
<resultlist state="c181538ad4f4c1be29e73f85b9237653">
<result project="Proj1" repository="standard" arch="i586" code="unpublished" state="unpublished">
<status package="000product" code="excluded"/>
</result>
Expand All @@ -49,9 +52,10 @@ my $response_published = '<resultlist state="c181538ad4f4c1be29e73f85b9237653">
<result project="Proj1" repository="images" arch="x86_64" code="published" state="published">
<status package="000product" code="disabled"/>
</result>
</resultlist>';

my $response_publishing = '<resultlist state="c181538ad4f4c1be29e73f85b9237651">
</resultlist>',
Proj2 => '
<!-- This project is still being published. -->
<resultlist state="c181538ad4f4c1be29e73f85b9237651">
<result project="Proj1" repository="standard" arch="i586" code="unpublished" state="unpublished">
<status package="000product" code="excluded"/>
</result>
Expand All @@ -67,10 +71,16 @@ my $response_publishing = '<resultlist state="c181538ad4f4c1be29e73f85b9237651">
<result project="Proj1" repository="images" arch="x86_64" code="published" state="published">
<status package="000product" code="disabled"/>
</result>
</resultlist>';

my $response_dirty = 'dirty';
our $response;
</resultlist>',
Proj1 => '
<!-- This project is "dirty". -->
<resultlist state="c181538ad4f4c1be29e73f85b9237653">
<result project="Proj1" repository="images" arch="x86_64" code="published" state="published" dirty>
<status package="000product" code="disabled"/>
</result>
</resultlist>',
Proj0 => 'invalid XML',
);

$SIG{INT} = sub {
session->clean;
Expand All @@ -84,22 +94,12 @@ my $host = "http://127.0.0.1:$port";
sub fake_api_server {
my $mock = Mojolicious->new;
$mock->mode('test');
$mock->routes->get(
'/public/build/Proj1/_result' => sub {
my $c = shift;
return $c->render(status => 200, text => $response_dirty);
});
$mock->routes->get(
'/public/build/Proj2/_result' => sub {
my $c = shift;
return $c->render(status => 200, text => $response_publishing);
});
$mock->routes->get(
'/public/build/Proj3/_result' => sub {
my $c = shift;
return $c->render(status => 200, text => $response_published);
});

for my $project (sort keys %fake_response_by_project) {
$mock->routes->get(
"/public/build/$project/_result" => sub {
shift->render(status => 200, text => $fake_response_by_project{$project});
});
}
return $mock;
}

Expand Down Expand Up @@ -154,14 +154,10 @@ $t->app->minion->on(
});

subtest 'test helper directly' => sub {
my $res = $t->app->obs_rsync->is_status_dirty('Proj1');
ok($res, "Status dirty");

$res = $t->app->obs_rsync->is_status_dirty('Proj2');
ok($res, "Status publishing $res");

$res = $t->app->obs_rsync->is_status_dirty('Proj3');
ok(!$res, "Status published $res");
is($t->app->obs_rsync->is_status_dirty('Proj0'), undef, 'status unknown');
is($t->app->obs_rsync->is_status_dirty('Proj1'), 1, 'status dirty');
is($t->app->obs_rsync->is_status_dirty('Proj2'), 1, 'status publishing');
is($t->app->obs_rsync->is_status_dirty('Proj3'), 0, 'status published');
};

$t->get_ok('/');
Expand Down