Skip to content

Commit

Permalink
Merge pull request #14010 from mlschroe/master
Browse files Browse the repository at this point in the history 
[backend] make use of a bininfocookie when fetching remote bininfo
  • Loading branch information
@mlschroe
mlschroe committed Mar 15, 2023
2 parents 017f59c + 2b9f635 commit b7a8324
Show file tree
Hide file tree
Showing 2 changed files with 161 additions and 66 deletions.
199 changes: 135 additions & 64 deletions src/backend/BSSched/Remote.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -535,12 +535,82 @@ sub addrepo_remote_unpackcpio {
return $r;
}

###########################################################################
###
### packstatus cache handling
###

sub update_remotepackstatus_cache {
my ($gctx, $prpa, $rpackstatus, $packstatususer) = @_;
return unless $packstatususer && $gctx->{'asyncmode'};
my $remotepackstatus = $gctx->{'remotepackstatus'};
my $remotepackstatus_cleanup = $gctx->{'remotepackstatus_cleanup'};
$rpackstatus->{'/users'} = [];
$rpackstatus->{'/users'} = [ @{$remotepackstatus->{$prpa}->{'/users'} || []} ] if $remotepackstatus->{$prpa};
push @{$rpackstatus->{'/users'}}, $packstatususer unless grep {$_ eq $packstatususer} @{$rpackstatus->{'/users'}};
push @{$remotepackstatus_cleanup->{$packstatususer}}, $prpa;
$remotepackstatus->{$prpa} = $rpackstatus;
}

sub read_remotepackstatus_cache {
my ($gctx, $prpa, $packstatususer) = @_;
return undef unless $packstatususer && $gctx->{'asyncmode'};
my $remotepackstatus = $gctx->{'remotepackstatus'};
my $ps = $remotepackstatus->{$prpa};
return undef unless $ps;
return undef unless grep {$_ eq $packstatususer} @{$ps->{'/users'} || []};
return $ps;
}

sub cleanup_remotepackstatus {
my ($gctx, $packstatususer) = @_;

my $remotepackstatus = $gctx->{'remotepackstatus'};
my $remotepackstatus_cleanup = $gctx->{'remotepackstatus_cleanup'};
return unless $remotepackstatus_cleanup->{$packstatususer};
print " cleaning up remote packstatus\n";
for my $prpa (@{$remotepackstatus_cleanup->{$packstatususer}}) {
my $rpackstatus = $remotepackstatus->{$prpa};
my @users = grep {$_ ne $packstatususer} @{$rpackstatus->{'/users'} || []};
$rpackstatus->{'/users'} = \@users;
print " - $prpa: ".@users." users\n";
delete $remotepackstatus->{$prpa} unless @users;
}
delete $remotepackstatus_cleanup->{$packstatususer};
}

###########################################################################
###
### remote BuildResult (aka gbininfo) support
###

sub read_gbininfo_remote_cache {
my ($gctx, $prpa, $packstatus, $rpackstatus) = @_;
my $remotecache = $gctx->{'remotecache'};
my $cachemd5 = Digest::MD5::md5_hex($prpa);
substr($cachemd5, 2, 0, '/');
return undef unless -s "$remotecache/$cachemd5.bininfo";
my $gbininfo = BSUtil::retrieve("$remotecache/$cachemd5.bininfo", 1);
return undef unless $gbininfo;
if ($packstatus && $rpackstatus) {
$packstatus->{$_} = $rpackstatus->{$_} for keys %$rpackstatus;
delete $packstatus->{'/users'};
}
return $gbininfo;
}

sub update_gbininfo_remote_cache {
my ($gctx, $prpa, $gbininfo, $cookie) = @_;
my $remotecache = $gctx->{'remotecache'};
my $cachemd5 = Digest::MD5::md5_hex($prpa);
substr($cachemd5, 2, 0, '/');
mkdir_p("$remotecache/".substr($cachemd5, 0, 2));
BSUtil::store("$remotecache/$cachemd5.bininfo.new$$", "$remotecache/$cachemd5.bininfo", $gbininfo);
my $remotegbininfos = $gctx->{'remotegbininfos'};
$remotegbininfos->{$prpa} = { 'lastfetch' => time() };
$remotegbininfos->{$prpa}->{'cookie'} = $cookie if defined $cookie;
}

sub read_gbininfo_remote {
my ($ctx, $prpa, $remoteproj, $packstatus) = @_;

Expand All @@ -552,9 +622,6 @@ sub read_gbininfo_remote {

my $gctx = $ctx->{'gctx'};
my $remotegbininfos = $gctx->{'remotegbininfos'};
my $cachemd5 = Digest::MD5::md5_hex($prpa);
substr($cachemd5, 2, 0, '/');

my $now = time();

# first check error case
Expand All @@ -563,32 +630,44 @@ sub read_gbininfo_remote {
return undef;
}

# check if we can use the cache
# check if we can use our packstatus cache
my $rpackstatus;
if ($packstatus) {
my $remotepackstatus = $gctx->{'remotepackstatus'};
if ($remotepackstatus->{$prpa} && $gctx->{'asyncmode'}) {
my $prp = $ctx->{'prp'};
$rpackstatus = $remotepackstatus->{$prpa} if grep {$_ eq $prp} @{$remotepackstatus->{$prpa}->{'/users'} || []};
}
}
$rpackstatus = read_remotepackstatus_cache($gctx, $prpa, $ctx->{'prp'}) if $packstatus;

if ((!$packstatus || $rpackstatus) && $remotegbininfos->{$prpa} && ($remotegbininfos->{$prpa}->{'lastfetch'} || 0) > $now - 3600) {
my $remotecache = $gctx->{'remotecache'};
if (-s "$remotecache/$cachemd5.bininfo") {
my $gbininfo = BSUtil::retrieve("$remotecache/$cachemd5.bininfo", 1);
if ($gbininfo) {
if ($packstatus) {
for my $pkg (keys %$gbininfo) {
$packstatus->{$pkg} = $rpackstatus->{$pkg} if $rpackstatus->{$pkg};
}
}
return $gbininfo;
}
my $gbininfo = read_gbininfo_remote_cache($gctx, $prpa, $packstatus, $rpackstatus);
return $gbininfo if $gbininfo;
}

# need to fetch new data, first try the packstatus if we have a bininfo cookie
my ($projid, $repoid, $arch) = split('/', $prpa, 3);
if ($remoteproj->{'partition'} && $remotegbininfos->{$prpa} && $remotegbininfos->{$prpa}->{'cookie'}) {
print " fetching remote packstatus for $prpa\n";
my $param = {
'uri' => "$remoteproj->{'remoteurl'}/build/$remoteproj->{'remoteproject'}/$repoid/$arch",
'timeout' => 200,
'proxy' => $gctx->{'remoteproxy'},
};
if ($gctx->{'asyncmode'}) {
$param->{'async'} = { '_resume' => \&read_packstatus_remote_resume, '_prpa' => $prpa };
}
my $rpackstatus;
eval {
$rpackstatus = $ctx->xrpc("bininfo/$prpa", $param, \&BSUtil::fromstorable, "view=packstatus", "withbininfocookie=1");
};
return 0 if !$@ && $rpackstatus && $param->{'async'};
if ($@) {
warn($@);
} elsif ($rpackstatus->{'.bininfocookie'} && $rpackstatus->{'.bininfocookie'} eq ($remotegbininfos->{$prpa}->{'cookie'} || '')) {
delete $rpackstatus->{'.bininfocookie'};
update_remotepackstatus_cache($gctx, $prpa, $rpackstatus, $ctx->{'prp'});
my $gbininfo = read_gbininfo_remote_cache($gctx, $prpa, $packstatus, $rpackstatus);
return $gbininfo if $gbininfo;
}
delete $remotegbininfos->{$prpa}->{'cookie'};
}

print " fetching remote project binary state for $prpa\n";
my ($projid, $repoid, $arch) = split('/', $prpa, 3);
my $param = {
'uri' => "$remoteproj->{'remoteurl'}/build/$remoteproj->{'remoteproject'}/$repoid/$arch",
'timeout' => 200,
Expand All @@ -601,7 +680,7 @@ sub read_gbininfo_remote {
eval {
if ($remoteproj->{'partition'}) {
$param->{'async'}->{'_isgbininfo'} = 1 if $param->{'async'};
$packagebinarylist = $ctx->xrpc("bininfo/$prpa", $param, \&BSUtil::fromstorable, "view=gbininfocode");
$packagebinarylist = $ctx->xrpc("bininfo/$prpa", $param, \&BSUtil::fromstorable, "view=gbininfocode", "withbininfocookie=1");
} else {
$packagebinarylist = $ctx->xrpc("bininfo/$prpa", $param, $BSXML::packagebinaryversionlist, "view=binaryversionscode");
}
Expand All @@ -620,21 +699,43 @@ sub read_gbininfo_remote {
if ($packstatus && $rpackstatus) {
$packstatus->{$_} = $rpackstatus->{$_} for keys %$rpackstatus;
delete $packstatus->{'/users'};
delete $packstatus->{'/bininfocookie'};
}
return $gbininfo;
}

sub read_packstatus_remote_resume {
my ($ctx, $handle, $error, $rpackstatus) = @_;
my $gctx = $ctx->{'gctx'};
my $prpa = $handle->{'_prpa'};
my $remotegbininfos = $gctx->{'remotegbininfos'};
if ($error) {
# on error just delete the cookie and retry
chomp $error;
warn("$error\n");
delete $remotegbininfos->{$prpa}->{'cookie'} if $remotegbininfos->{$prpa};
$ctx->setchanged_unless_othersinprogress($handle);
return;
}
if ($rpackstatus->{'.bininfocookie'} && $rpackstatus->{'.bininfocookie'} eq ($remotegbininfos->{$prpa}->{'cookie'} || '') && $ctx->{'prp'}) {
# gbininfo matches! update cache
update_remotepackstatus_cache($gctx, $prpa, $rpackstatus, $ctx->{'prp'});
} else {
# sorry, need to do fetch the complete gbininfo. delete the cookie and retry
delete $remotegbininfos->{$prpa}->{'cookie'} if $remotegbininfos->{$prpa};
}
$ctx->setchanged_unless_othersinprogress($handle);
}

sub read_gbininfo_remote_resume {
my ($ctx, $handle, $error, $packagebinarylist) = @_;
my $gctx = $ctx->{'gctx'};
convertpackagebinarylist($gctx, $handle->{'_prpa'}, $packagebinarylist, $error, $ctx->{'prp'}, $handle->{'_isgbininfo'});
convertpackagebinarylist($ctx->{'gctx'}, $handle->{'_prpa'}, $packagebinarylist, $error, $ctx->{'prp'}, $handle->{'_isgbininfo'});
$ctx->setchanged_unless_othersinprogress($handle);
}

sub convertpackagebinarylist {
my ($gctx, $prpa, $packagebinarylist, $error, $packstatususer, $isgbininfo) = @_;

my $remotegbininfos = $gctx->{'remotegbininfos'};
if ($error) {
chomp $error;
warn("$error\n");
Expand All @@ -643,13 +744,15 @@ sub convertpackagebinarylist {
my ($projid, $repoid, $arch) = split('/', $prpa, 3);
$gctx->{'retryevents'}->addretryevent({'type' => 'scanprjbinaries', 'project' => $projid, 'repository' => $repoid, 'arch' => $arch});
}
$remotegbininfos->{$prpa} = { 'lastfetch' => time(), 'error' => $error };
$gctx->{'remotegbininfos'}->{$prpa} = { 'lastfetch' => time(), 'error' => $error };
return (undef, undef);
}
my $gbininfo = {};
my $rpackstatus = {};
my $bininfocookie;
if ($isgbininfo) {
$gbininfo = $packagebinarylist || {};
$bininfocookie = delete $gbininfo->{'.bininfocookie'};
for my $pkg (keys %$gbininfo) {
my $bi = $gbininfo->{$pkg};
$rpackstatus->{$pkg} = delete($bi->{'.code'}) if exists $bi->{'.code'};
Expand All @@ -669,52 +772,20 @@ sub convertpackagebinarylist {
} elsif ($filename eq '.nouseforbuild') {
$bins{$filename} = {};
} else {
$bins{$filename} = {'filename' => $filename}; # XXX: what about the md5sum for appdata/modulemd?
$bins{$filename} = {'filename' => $filename};
}
$bins{$filename}->{'hdrmd5'} = $binary->{'hdrmd5'} if $binary->{'hdrmd5'};
$bins{$filename}->{'leadsigmd5'} = $binary->{'leadsigmd5'} if $binary->{'leadsigmd5'};
$bins{$filename}->{'md5sum'} = $binary->{'md5sum'} if $binary->{'md5sum'};
}
my $pkg = $binaryversionlist->{'package'};
$gbininfo->{$pkg} = \%bins;
$rpackstatus->{$pkg} = $binaryversionlist->{'code'} if $binaryversionlist->{'code'};
}
}
my $remotecache = $gctx->{'remotecache'};
my $cachemd5 = Digest::MD5::md5_hex($prpa);
substr($cachemd5, 2, 0, '/');
mkdir_p("$remotecache/".substr($cachemd5, 0, 2));
BSUtil::store("$remotecache/$cachemd5.bininfo.new$$", "$remotecache/$cachemd5.bininfo", $gbininfo);

$remotegbininfos->{$prpa} = { 'lastfetch' => time() };

if ($packstatususer) {
my $remotepackstatus = $gctx->{'remotepackstatus'};
my $remotepackstatus_cleanup = $gctx->{'remotepackstatus_cleanup'};
$rpackstatus->{'/users'} = [];
$rpackstatus->{'/users'} = [ @{$remotepackstatus->{$prpa}->{'/users'} || []} ] if $remotepackstatus->{$prpa};
push @{$rpackstatus->{'/users'}}, $packstatususer unless grep {$_ eq $packstatususer} @{$rpackstatus->{'/users'}};
push @{$remotepackstatus_cleanup->{$packstatususer}}, $prpa;
$remotepackstatus->{$prpa} = $rpackstatus;
}

update_gbininfo_remote_cache($gctx, $prpa, $gbininfo, $bininfocookie);
update_remotepackstatus_cache($gctx, $prpa, $rpackstatus, $packstatususer);
return ($gbininfo, $rpackstatus);
}

sub cleanup_remotepackstatus {
my ($gctx, $prp) = @_;

my $remotepackstatus = $gctx->{'remotepackstatus'};
my $remotepackstatus_cleanup = $gctx->{'remotepackstatus_cleanup'};
return unless $remotepackstatus_cleanup->{$prp};
print " cleaning up remote packstatus\n";
for my $prpa (@{$remotepackstatus_cleanup->{$prp}}) {
my $rpackstatus = $remotepackstatus->{$prpa};
my @users = grep {$_ ne $prp} @{$rpackstatus->{'/users'} || []};
$rpackstatus->{'/users'} = \@users;
print " - $prpa: ".@users." users\n";
delete $remotepackstatus->{$prpa} unless @users;
}
delete $remotepackstatus_cleanup->{$prp};
}

1;
28 changes: 26 additions & 2 deletions src/backend/bs_repserver
View file
Original file line number Diff line number Diff line change
Expand Up @@ -302,18 +302,41 @@ sub getpackagebinaryversionlist {
return ({ 'binaryversionlist' => \@res }, $BSXML::packagebinaryversionlist);
}

sub calc_bininfocookie {
my ($prpa) = @_;
my @s2 = stat("$reporoot/$prpa/:bininfo.merge");
my @s1 = stat("$reporoot/$prpa/:bininfo");
my $cookie = @s1 ? "$s1[9]/$s1[7]/$s1[1]" : '0/0/0';
$cookie .= "+$s2[9]/$s2[7]/$s2[1]" if @s2;
return $cookie;
}

sub getpackstatus {
my ($cgi, $projid, $repoid, $arch) = @_;
my $prp = "$projid/$repoid";

die("getpackstatus: package filtering is not supported\n") if $cgi->{'package'};
my $cookie = $cgi->{'withbininfocookie'} ? calc_bininfocookie("$prp/$arch") : undef;
my $ps = readpackstatus("$prp/$arch");
$ps = ($ps || {})->{'packstatus'} || {};
$ps->{'.bininfocookie'} = $cookie if defined $cookie;
return (BSUtil::tostorable($ps), 'Content-Type: application/octet-stream');
}

sub getgbininfo {
my ($cgi, $projid, $repoid, $arch) = @_;
my $prp = "$projid/$repoid";

die("getgbininfo: package filtering is not supported\n") if $cgi->{'package'};
my $cookie = $cgi->{'withbininfocookie'} ? calc_bininfocookie("$prp/$arch") : undef;
my $gbininfo = BSRepServer::read_gbininfo("$reporoot/$prp/$arch", 1, 1);
if ($cgi->{'withcode'}) {
my $ps = readpackstatus("$projid/$repoid/$arch");
my $ps = readpackstatus("$prp/$arch");
my $code = ($ps || {})->{'packstatus'} || {};
$gbininfo->{$_}->{'.code'} = $code->{$_} for keys %$code;
}
delete $_->{'.bininfo'} for values %$gbininfo;
$gbininfo->{'.bininfocookie'} = $cookie if defined $cookie;
return (BSUtil::tostorable($gbininfo), 'Content-Type: application/octet-stream');
}

Expand Down Expand Up @@ -363,6 +386,7 @@ sub getpackagelist_build {
return getbinarychecksums($cgi, $projid, $repoid, $arch) if $view eq 'binarychecksums' || $view eq 'rawbinarychecksums';
$cgi->{'withcode'} = 1 if $view eq 'binaryversionscode' || $view eq 'gbininfocode';
return getgbininfo($cgi, $projid, $repoid, $arch) if $view eq 'gbininfo' || $view eq 'gbininfocode';
return getpackstatus($cgi, $projid, $repoid, $arch) if $view eq 'packstatus';
return getpackagebinaryversionlist($cgi, $projid, $repoid, $arch, $cgi->{'package'}) if $view eq 'binaryversions' || $view eq 'binaryversionscode';
die("unknown view '$view'\n");
}
Expand Down Expand Up @@ -4413,7 +4437,7 @@ my $dispatches = [
'POST:/build/$project cmd=move oproject:project' => \&moveproject,
'POST:/build/$project/$repository/$arch/_repository cmd=missingdodresources resource:* partition:?' => \&missingdodresources,
'POST:/build/$project/$repository/$arch/_repository match:' => \&postrepo,
'/build/$project/$repository/$arch package* view:?' => \&getpackagelist_build,
'/build/$project/$repository/$arch package* view:? withbininfocookie:bool?' => \&getpackagelist_build,
'/build/$project/$repository/$arch/_builddepinfo package* view:?' => \&getbuilddepinfo,
'/build/$project/$repository/$arch/_jobhistory package* code:* limit:num? endtime_start:num? endtime_end:num?' => \&getjobhistory,
'POST:/build/$project/$repository/$arch/_relsync' => \&postrelsync,
Expand Down

0 comments on commit b7a8324

Please sign in to comment.