Skip to content

Commit

Permalink
Fix mirrors report when missing folders
Browse files Browse the repository at this point in the history
  • Loading branch information
@andrii-suse
andrii-suse committed Jul 12, 2022
1 parent 7a53cce commit 7d35958
Show file tree
Hide file tree
Showing 3 changed files with 35 additions and 33 deletions.
30 changes: 14 additions & 16 deletions lib/MirrorCache/Schema/ResultSet/Project.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,18 +13,17 @@ sub mirror_summary {
my $dbh = $self->result_source->schema->storage->dbh;

my $sql = <<"END_SQL";
select sum(case when diff = 0 then 1 else 0 end) as current, sum(case when diff = 0 then 0 else 1 end) as outdated
select sum(case when diff = 0 then 1 else 0 end) as current, sum(case when diff = 0 or diff is null then 0 else 1 end) as outdated
from (
select server_id, max(case when d1 <> d2 then 1 else 0 end) diff
select server_id, max(case when d1 <> d2 or d1 is null then 1 else 0 end) diff
from (
select prj.etalon, f.path, f.id, fd.folder_id, fds.server_id, fds.folder_diff_id as d1, fds2.folder_diff_id as d2
from project prj
select s.id as server_id, prj.name, fds.folder_diff_id d1, fds2.folder_diff_id d2
from server s
join project prj on prj.name = ?
join folder f on f.path like concat(prj.path,'/%')
join folder_diff fd on fd.folder_id = f.id
join folder_diff fd2 on fd2.folder_id = fd.folder_id
join folder_diff_server fds on fd.id = fds.folder_diff_id
join folder_diff_server fds2 on fd2.id = fds2.folder_diff_id and fds2.server_id = prj.etalon
where prj.name = ?
left join folder_diff_server fds on fds.folder_diff_id = fd.id and fds.server_id = s.id
join folder_diff_server fds2 on fds2.folder_diff_id = fd.id and fds2.server_id = prj.etalon
) x
group by server_id) xx
END_SQL
Expand All @@ -38,19 +37,18 @@ sub mirror_list {
my $dbh = $self->result_source->schema->storage->dbh;

my $sql = <<"END_SQL";
select server_id, min(case when d1 = d2 then 1 else 0 end) as current, concat(s.hostname, s.urldir) as url
select s.id server_id, min(case when d1 = d2 then 1 else 0 end) as current, concat(s.hostname, s.urldir) as url
from (
select fds.server_id, fds.folder_diff_id as d1, fds2.folder_diff_id as d2
from project prj
select s.id as server_id, prj.name, fds.folder_diff_id d1, fds2.folder_diff_id d2
from server s
join project prj on prj.name = ?
join folder f on f.path like concat(prj.path,'/%')
join folder_diff fd on fd.folder_id = f.id
join folder_diff fd2 on fd2.folder_id = fd.folder_id
join folder_diff_server fds on fd.id = fds.folder_diff_id
join folder_diff_server fds2 on fd2.id = fds2.folder_diff_id and fds2.server_id = prj.etalon
where prj.name = ?
left join folder_diff_server fds on fds.folder_diff_id = fd.id and fds.server_id = s.id
join folder_diff_server fds2 on fds2.folder_diff_id = fd.id and fds2.server_id = prj.etalon
) x
join server s on x.server_id = s.id
group by x.server_id, s.id
group by s.id, s.hostname, s.urldir
order by current desc, url
END_SQL
my $prep = $dbh->prepare($sql);
Expand Down
22 changes: 12 additions & 10 deletions lib/MirrorCache/Schema/ResultSet/Server.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -253,34 +253,36 @@ sub report_mirrors {
my $schema = $rsource->schema;
my $dbh = $schema->storage->dbh;

my $whereon = 'ON 1 = 1';
my $where1 = '';
my $where2 = '';
$whereon = "ON REPLACE(prj.name,' ','') = ?" if $project;
$where1 = "WHERE REPLACE(prj.name,' ','') = ?" if $project;
$where2 = "WHERE s.region = ?" if $region;

my $sql = <<"END_SQL";
select s.region, s.country, x.name project, s.id, min(case when s1 = 0 then 1 else 0 end) as curr, round((sum(s2)*1.0)/(sum(s1)+sum(s2))*100) as score, concat(s.hostname, s.urldir) as url, max(example) victim
from (
select prj.name, fds.server_id,
sum(case when fds.folder_diff_id != fds2.folder_diff_id then 1 else 0 end) s1,
sum(case when fds.folder_diff_id = fds2.folder_diff_id then 1 else 0 end) s2,
select prj.name, s.id,
sum(case when fds.folder_diff_id != fds2.folder_diff_id or fds.folder_diff_id is null then 1 else 0 end) s1,
sum(case when fds.folder_diff_id = fds2.folder_diff_id and fds.folder_diff_id is not null then 1 else 0 end) s2,
max(case when fds.folder_diff_id != fds2.folder_diff_id then f.path else '' end) example
from project prj
from server s
join project prj $whereon
join folder f on f.path like concat(prj.path,'%')
join folder_diff fd on fd.folder_id = f.id
join folder_diff fd2 on fd2.folder_id = fd.folder_id
join folder_diff_server fds on fd.id = fds.folder_diff_id
join folder_diff_server fds2 on fd2.id = fds2.folder_diff_id and fds2.server_id = prj.etalon
left join folder_diff_server fds on fd.id = fds.folder_diff_id and fds.server_id = s.id
join folder_diff_server fds2 on fd.id = fds2.folder_diff_id and fds2.server_id = prj.etalon
$where1
group by fds2.server_id, prj.name, fds.server_id ) x
join server s on x.server_id = s.id
group by s.id, prj.name, fds.server_id ) x
join server s on x.id = s.id
$where2
group by s.id, x.name
order by region, country, score, url;
END_SQL
my $prep = $dbh->prepare($sql);
if ($project && $region) {
$prep->execute($project, $region);
$prep->execute($project, $project, $region);
} elsif ($project) {
$prep->execute($project);
} elsif ($region) {
Expand Down
16 changes: 9 additions & 7 deletions t/environ/14-project.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,9 @@ $mc/start
ap8=$(environ ap8)
ap7=$(environ ap7)
ap6=$(environ ap6)
ap5=$(environ ap5)

for x in $mc $ap7 $ap8 $ap6; do
for x in $mc $ap7 $ap8 $ap6 $ap5; do
mkdir -p $x/dt/{folder1,folder2,folder3}
mkdir -p $x/dt/project1/{folder1,folder2,folder3}
mkdir -p $x/dt/project2/{folder1,folder2,folder3}
Expand All @@ -18,16 +19,19 @@ for x in $mc $ap7 $ap8 $ap6; do
echo $x/dt/project2/{folder1,folder2,folder3}/{file1.1,file2.1}.dat | xargs -n 1 touch
done

$ap5/start
$ap6/start
$ap7/start
$ap8/start

# remove a file from ap7
rm $ap7/dt/project1/folder2/file2.1.dat
rm -r $ap5/dt/project1/folder2/

$mc/sql "insert into server(hostname,urldir,enabled,country,region) select '$($ap6/print_address)','','t','us','na'"
$mc/sql "insert into server(hostname,urldir,enabled,country,region) select '$($ap7/print_address)','','t','us','na'"
$mc/sql "insert into server(hostname,urldir,enabled,country,region) select '$($ap8/print_address)','','t','de','eu'"
$mc/sql "insert into server(hostname,urldir,enabled,country,region) select '$($ap5/print_address)','','t','as','cn'"

$mc/sql "insert into project(name,path,etalon) select 'proj1','/project1', 3"
$mc/sql "insert into project(name,path,etalon) select 'proj 2','/project2', 3"
Expand All @@ -39,8 +43,7 @@ $mc/backstage/shoot

$mc/curl -s /rest/project/proj1
$mc/curl -s /rest/project/proj1/mirror_summary
$mc/curl -s /rest/project/proj1/mirror_summary | grep -E '"current":"?3' | grep -E '"outdated":"?0'

$mc/curl -s /rest/project/proj1/mirror_summary | grep -E '"current":"?4' | grep -E '"outdated":"?0'

$mc/backstage/job -e folder_sync -a '["/project1/folder2"]'
$mc/backstage/job -e mirror_scan -a '["/project1/folder2"]'
Expand All @@ -53,10 +56,9 @@ $mc/backstage/shoot
$mc/backstage/job -e report -a '["once"]'
$mc/backstage/shoot

$mc/curl -s /rest/repmirror | grep '[{"country":"de","proj1score":"100","proj1victim":"","proj2score":"100","proj2victim":"","region":"eu","url":"127.0.0.1:1314"},{"country":"us","proj1score":"100","proj1victim":"","proj2score":"100","proj2victim":"","region":"na","url":"127.0.0.1:1294"},{"country":"us","proj1score":"50","proj1victim":"\/project1\/folder2","proj2score":"100","proj2victim":"","region":"na","url":"127.0.0.1:1304"}]'
$mc/curl -s /rest/repmirror | grep -F '{"country":"as","proj1score":"50","proj1victim":"","proj2score":"100","proj2victim":"","region":"cn","url":"127.0.0.1:1284"},{"country":"de","proj1score":"100","proj1victim":"","proj2score":"100","proj2victim":"","region":"eu","url":"127.0.0.1:1314"},{"country":"us","proj1score":"100","proj1victim":"","proj2score":"100","proj2victim":"","region":"na","url":"127.0.0.1:1294"},{"country":"us","proj1score":"50","proj1victim":"","proj2score":"100","proj2victim":"","region":"na","url":"127.0.0.1:1304"}'

$mc/curl -s /rest/project/proj1/mirror_summary
$mc/curl -s /rest/project/proj1/mirror_summary | grep -E '"current":"?2' | grep -E '"outdated":"?1'
$mc/curl -s /rest/project/proj1/mirror_summary | grep -E '"current":"?2' | grep -E '"outdated":"?2'

# $mc/curl -s /rest/project/proj1/mirror_list | grep -E '{"current":"?1"?,"server_id":"?1"?,"url":"127.0.0.1:1294"},{"current":"?1"?,"server_id":"?3"?,"url":"127.0.0.1:1314"},{"current":"?0"?,"server_id":"?2"?,"url":"127.0.0.1:1304"}'
$mc/curl -s /rest/project/proj1/mirror_list | grep '{"current":1,"server_id":1,"url":"127.0.0.1:1294"},{"current":1,"server_id":3,"url":"127.0.0.1:1314"},{"current":0,"server_id":2,"url":"127.0.0.1:1304"}'
$mc/curl -s /rest/project/proj1/mirror_list | grep -E '{"current":"?1"?,"server_id":"?1"?,"url":"127.0.0.1:1294"},{"current":"?1"?,"server_id":"?3"?,"url":"127.0.0.1:1314"},{"current":"?0"?,"server_id":"?4"?,"url":"127.0.0.1:1284"},{"current":"?0"?,"server_id":"?2"?,"url":"127.0.0.1:1304"}'

0 comments on commit 7d35958

Please sign in to comment.