Skip to content

Commit

Permalink
Merge pull request #6453 from mlschroe/master
Browse files Browse the repository at this point in the history 
Report weakdeps in fileinfo queries
  • Loading branch information
@mlschroe
mlschroe committed Dec 5, 2018
2 parents 65a72cc + b0b6b5f commit 914b475
Show file tree
Hide file tree
Showing 9 changed files with 129 additions and 83 deletions.
10 changes: 7 additions & 3 deletions src/backend/BSRepServer/Checker.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -123,9 +123,13 @@ sub preparepool {
}
$ctx->{'dep2pkg'} = \%dep2pkg;
$ctx->{'dep2src'} = \%dep2src;
my @subpacks = grep {defined($dep2src{$_}) && $dep2src{$_} eq $pname} keys %dep2src;
@subpacks = () if $bconf->{'type'} eq 'kiwi' || $bconf->{'type'} eq 'docker';
$ctx->{'subpacks'} = { $pname => \@subpacks };
if (!defined($pname)) {
$ctx->{'subpacks'} = {};
} else {
my @subpacks = grep {defined($dep2src{$_}) && $dep2src{$_} eq $pname} keys %dep2src;
@subpacks = () if $bconf->{'type'} eq 'kiwi' || $bconf->{'type'} eq 'docker';
$ctx->{'subpacks'} = { $pname => \@subpacks };
}
}

# see checkpks in BSSched::Checker
Expand Down
2 changes: 2 additions & 0 deletions src/backend/BSRevision.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ use BSUtil;
use BSFileDB;
use BSSrcrep;
use BSDB;
use Storable;

my $projectsdir = "$BSConfig::bsdir/projects";
my $srcrep = "$BSConfig::bsdir/sources";
Expand Down Expand Up @@ -575,6 +576,7 @@ sub readproj_local {
$proj = readxml("$projectsdir/$projid.xml", $BSXML::proj, 1);
$readproj_local_cache->{$projid} = $proj;
}
$proj = Storable::dclone($proj) if $proj;
} else {
$proj = readxml("$projectsdir/$projid.xml", $BSXML::proj, 1);
}
Expand Down
24 changes: 12 additions & 12 deletions src/backend/BSSched/BuildJob.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -888,7 +888,7 @@ sub create {
if ($kiwimode) {
# switch searchpath to kiwi info path
$syspath = $searchpath if @$searchpath;
$searchpath = path2buildinfopath($gctx, [ expandkiwipath($info, $ctx->{'prpsearchpath'}) ]);
$searchpath = path2buildinfopath($gctx, [ expandkiwipath($ctx, $info) ]);
}

my $expanddebug = $ctx->{'expanddebug'};
Expand Down Expand Up @@ -1232,28 +1232,28 @@ sub diffsortedmd5 {
return @ret;
}

=head2 expandkiwipath - TODO: add summary
=head2 expandkiwipath - turn the path from the info into a kiwi searchpath
TODO: add description
=cut

sub expandkiwipath {
my ($info, $prpsearchpath, $prios) = @_;
my ($ctx, $info, $prios) = @_;
my @path;
for (@{$info->{'path'} || []}) {
if ($_->{'project'} eq '_obsrepositories') {
push @path, @{$prpsearchpath || []};
} else {
my $prp = "$_->{'project'}/$_->{'repository'}";
push @path, $prp;
if ($prios) {
my $prio = $_->{'priority'} || 0;
$prios->{$prp} = $prio if !defined($prios->{$prp}) || $prio > $prios->{$prp};
}
push @path, @{$ctx->{'prpsearchpath'} || []};
next;
}
my $prp = "$_->{'project'}/$_->{'repository'}";
push @path, $prp;
if ($prios) {
my $prio = $_->{'priority'} || 0;
$prios->{$prp} = $prio if !defined($prios->{$prp}) || $prio > $prios->{$prp};
}
}
return @path;
return BSUtil::unify(@path);
}

=head2 getcontainerannotation - get the annotation from a container package
Expand Down
5 changes: 3 additions & 2 deletions src/backend/BSSched/BuildJob/Docker.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -142,8 +142,9 @@ sub check {

# append container repositories to path
my @newpath;
my $haveobsrepositories = grep {$_->{'project'} eq '_obsrepositories'} @{$info->{'path'} || []};
my $annotation = BSSched::BuildJob::getcontainerannotation($cpool, $p, $cbdep);
if ($annotation) {
if ($annotation && !$haveobsrepositories) {
# map all repos and add to path
my $remoteprojs = $gctx->{'remoteprojs'} || {};
my $rproj = $remoteprojs->{(split('/', $cprp, 2))[0]};
Expand Down Expand Up @@ -183,7 +184,7 @@ sub check {


my %aprpprios;
my @aprps = BSSched::BuildJob::expandkiwipath($info, $ctx->{'prpsearchpath'}, %aprpprios);
my @aprps = BSSched::BuildJob::expandkiwipath($ctx, $info, \%aprpprios);

# get config from docker path
my @configpath = @aprps;
Expand Down
2 changes: 1 addition & 1 deletion src/backend/BSSched/BuildJob/KiwiImage.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ sub check {
my $neverblock = $ctx->{'isreposerver'} || ($repo->{'block'} || '' eq 'never');

my %aprpprios;
my @aprps = BSSched::BuildJob::expandkiwipath($info, $ctx->{'prpsearchpath'}, \%aprpprios);
my @aprps = BSSched::BuildJob::expandkiwipath($ctx, $info, \%aprpprios);
# get config from kiwi path
my @configpath = @aprps;
# always put ourselfs in front
Expand Down
2 changes: 1 addition & 1 deletion src/backend/BSSched/BuildJob/KiwiProduct.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,7 @@ sub check {
my %deps = map {$_ => 1} @deps;
delete $deps{''};

my @aprps = BSSched::BuildJob::expandkiwipath($info, $ctx->{'prpsearchpath'});
my @aprps = BSSched::BuildJob::expandkiwipath($ctx, $info);
my @bprps = @{$ctx->{'prpsearchpath'}};
my $bconf = $ctx->{'conf'};

Expand Down
29 changes: 20 additions & 9 deletions src/backend/BSXML.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -414,6 +414,17 @@ our $dir = [
]]
];

our $providedby = [
'providedby' =>
'name',
'epoch',
'version',
'release',
'arch',
'project',
'repository',
];

our $fileinfo = [
'fileinfo' =>
'filename',
Expand Down Expand Up @@ -452,15 +463,15 @@ our $fileinfo = [
]],
[[ 'requires_ext' =>
'dep',
[[ 'providedby' =>
'name',
'epoch',
'version',
'release',
'arch',
'project',
'repository',
]],
[ $providedby ],
]],
[[ 'recommends_ext' =>
'dep',
[ $providedby ],
]],
[[ 'supplements_ext' =>
'dep',
[ $providedby ],
]],
];

Expand Down
115 changes: 69 additions & 46 deletions src/backend/bs_repserver
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1207,6 +1207,17 @@ sub getlogfile {
return undef;
}

sub add_provided_by {
my ($n, $pool, $dep, $keep) = @_;

for my $p ($pool->whatprovides($dep)) {
my $rd = $pool->pkg2data($p);
delete $rd->{$_} for grep {!$keep->{$_}} keys %$rd;
($rd->{'project'}, $rd->{'repository'}) = split('/', $pool->pkg2reponame($p), 2);
push @{$n->{'providedby'}}, $rd;
}
}

sub getbinary_info {
my ($cgi, $projid, $repoid, $arch, $bin, $path) = @_;
if ($path =~ /\.tar$/ && ! -f $path) {
Expand All @@ -1220,7 +1231,7 @@ sub getbinary_info {
}
my @s = stat($path);
die("404 $bin: $!\n") unless @s;
my $res = Build::query($path, 'evra' => 1, 'description' => 1) || {};
my $res = Build::query($path, 'evra' => 1, 'description' => 1, 'weakdeps' => 1) || {};
if (!%$res && $path =~ /\/updateinfo\.xml$/) {
my $updateinfos = readxml($path, $BSXML::updateinfo, 1);
if ($updateinfos && @{$updateinfos->{'update'} || []} == 1) {
Expand Down Expand Up @@ -1295,25 +1306,30 @@ sub getbinary_info {
}
$pool->createwhatprovides();
my %keep = map {$_ => 1} qw{name epoch version release arch};
for my $prov (@{$res->{'provides'}}) {
my $n = {'dep' => $prov};
push @{$res->{'provides_ext'}}, $n;
for my $p ($pool->whatrequires($prov)) {
for my $dep (@{$res->{'provides'} || []}) {
my $n = {'dep' => $dep};
for my $p ($pool->whatrequires($dep)) {
my $rd = $pool->pkg2data($p);
delete $rd->{$_} for grep {!$keep{$_}} keys %$rd;
($rd->{'project'}, $rd->{'repository'}) = split('/', $pool->pkg2reponame($p), 2);
push @{$n->{'requiredby'}}, $rd;
}
push @{$res->{'provides_ext'}}, $n;
}
for my $req (@{$res->{'requires'}}) {
my $n = {'dep' => $req};
for my $dep (@{$res->{'requires'} || []}) {
my $n = {'dep' => $dep};
add_provided_by($n, $pool, $dep, \%keep);
push @{$res->{'requires_ext'}}, $n;
for my $p ($pool->whatprovides($req)) {
my $rd = $pool->pkg2data($p);
delete $rd->{$_} for grep {!$keep{$_}} keys %$rd;
($rd->{'project'}, $rd->{'repository'}) = split('/', $pool->pkg2reponame($p), 2);
push @{$n->{'providedby'}}, $rd;
}
}
for my $dep (@{$res->{'recommends'} || []}) {
my $n = {'dep' => $dep};
add_provided_by($n, $pool, $dep, \%keep);
push @{$res->{'recommends_ext'}}, $n;
}
for my $dep (@{$res->{'supplements'} || []}) {
my $n = {'dep' => $dep};
add_provided_by($n, $pool, $dep, \%keep);
push @{$res->{'supplements_ext'}}, $n;
}
}
data2utf8xml($res);
Expand Down Expand Up @@ -3001,48 +3017,55 @@ sub getbuildinfo {
return ($binfo, $BSXML::buildinfo);
}

sub getbuildinfo_post_cpio {
my ($cgi, $projid, $repoid, $arch, $packid, $fp) = @_;

my $fn = $cgi->{'_fn'};
local *F;
open(F, '<', $fn) || die("$fn: $!\n");
unlink($fn);
my $dir = "$uploaddir/$$.dir";
mkdir_p($dir);
my $uploaded = BSHTTP::cpio_receiver(BSHTTP::fd2req(\*F), {'directory' => $dir});
close(F);
# should we check if the cpio archive contains <= 2 files?
my $depfile = (grep { $_->{'name'} eq 'deps' } @$uploaded)[0];
$depfile = "$dir/$depfile->{'name'}" if $depfile;
my $servicefile = (grep { $_->{'name'} eq '_service' } @$uploaded)[0];
$servicefile = "$dir/$servicefile->{'name'}" if $servicefile;
my $bifile = (grep { $_->{'name'} eq 'buildenv' } @$uploaded)[0];
$bifile = "$dir/$bifile->{'name'}" if $bifile;
$fn = (grep { $_->{'name'} ne 'deps' && $_->{'name'} ne 'buildenv' && $_->{'name'} ne '_service'} @$uploaded)[0];
die("no build recipe file found\n") unless $fn;
my @r;
eval {
@r = getbuildinfo_post({ %$cgi, '_fn' => "$dir/$fn->{'name'}", '_depfile' => $depfile, '_buildenvfile' => $bifile, '_servicefile' => $servicefile, '_buildtype' => Build::recipe2buildtype($fn->{'name'})}, $projid, $repoid, $arch, $packid);
};
# cleanup
unlink("$dir/$_") for ls($dir);
rmdir($dir) if -d $dir;
die("$@\n") if $@;
return @r;
}

sub getbuildinfo_post {
my ($cgi, $projid, $repoid, $arch, $packid) = @_;

my $fn = $cgi->{'_fn'};
my $depfile = $cgi->{'_depfile'};
if (!$fn) {
# get POST body
mkdir_p($uploaddir);
$fn = "$uploaddir/$$";
die("upload failed\n") unless BSServer::read_file($fn);
}

local *F;
open(F, '<', $fn) || die("$fn: $!\n");
my $magic;
sysread(F, $magic, 6);
if ($magic eq "070701" && !$cgi->{'_fn'}) {
# have cpio archive, extract recipe and depfile, recurse
unlink($fn);
sysseek(F, 0, 0);
my $dir = "$uploaddir/$$.dir";
mkdir_p($dir);
my $uploaded = BSHTTP::cpio_receiver(BSHTTP::fd2req(\*F), {'directory' => $dir});
# check if the upload is a cpio archive by looking at the first 6 bytes
local *F;
open(F, '<', $fn) || die("$fn: $!\n");
my $magic;
sysread(F, $magic, 6);
close(F);
# should we check if the cpio archive contains <= 2 files?
$depfile = (grep { $_->{'name'} eq 'deps' } @$uploaded)[0];
$depfile = "$dir/$depfile->{'name'}" if $depfile;
my $servicefile = (grep { $_->{'name'} eq '_service' } @$uploaded)[0];
$servicefile = "$dir/$servicefile->{'name'}" if $servicefile;
my $bifile = (grep { $_->{'name'} eq 'buildenv' } @$uploaded)[0];
$bifile = "$dir/$bifile->{'name'}" if $bifile;
$fn = (grep { $_->{'name'} ne "deps" && $_->{'name'} ne 'buildenv' && $_->{'name'} ne '_service'} @$uploaded)[0];
die("no build recipe file found\n") unless $fn;
my @r;
eval {
@r = getbuildinfo_post({ %$cgi, '_fn' => "$dir/$fn->{'name'}", '_depfile' => $depfile, '_buildenvfile' => $bifile, '_servicefile' => $servicefile, '_buildtype' => Build::recipe2buildtype($fn->{'name'})}, $projid, $repoid, $arch, $packid);
};
unlink("$dir/$_") for ls($dir);
rmdir($dir) if -d $dir;
die("$@\n") if $@;
return @r;
return getbuildinfo_post_cpio({ %$cgi, '_fn' => $fn }, $projid, $repoid, $arch, $packid) if $magic eq '070701';
}
close(F);

my $bconf = BSRepServer::getconfig($projid, $repoid, $arch);
$bconf->{'type'} = $cgi->{'_buildtype'} if $cgi->{'_buildtype'};
Expand Down Expand Up @@ -3090,7 +3113,7 @@ sub getbuildinfo_post {
push @{$info->{'dep'}}, $pkgname;
}
}
$pdata->{'ldepfile'} = $depfile if defined $depfile;
$pdata->{'ldepfile'} = $cgi->{'_depfile'} if $cgi->{'_depfile'};

my $binfo = BSRepServer::BuildInfo::buildinfo($projid, $repoid, $arch, $packid,
pdata => $pdata,
Expand Down Expand Up @@ -3629,7 +3652,7 @@ sub fileinfo {
my $q = {};
die("filename: $!\n") unless -f $filepath;
if ($filename =~ /\.(?:$binsufsre)$/) {
$q = Build::query($filepath, 'evra' => 1, 'description' => 1, 'alldeps' => 1);
$q = Build::query($filepath, 'evra' => 1, 'description' => 1, 'weakdeps' => 1, 'alldeps' => 1);
data2utf8xml($q);
} elsif ($filename =~ /\.ymp$/) {
my $ymp = readxml($filepath, $BSXML::ymp, 1);
Expand Down
23 changes: 14 additions & 9 deletions src/backend/t/0100-BSSched-BuildJob.t
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,42 +58,47 @@ for my $got (@$changes) {
};

################################################################################
$fixture = {
$fixture = [
{},
{
path => [
{ path => 'abc' , project => 'openSUSE:Factory' , repository => 'standard' },
],
};
},
];

@$got = BSSched::BuildJob::expandkiwipath($fixture);
@$got = BSSched::BuildJob::expandkiwipath(@$fixture);
$expected = [ 'openSUSE:Factory/standard' ];
is_deeply($got,$expected,'Checking testcase 1 TODO: better description');
################################################################################
$fixture = [
{
prpsearchpath => ['prpsearchpath1','prpsearchpath2']
},
{
path => [
{ path => 'abc' , project => '_obsrepositories' },
],
},
['prpsearchpath1','prpsearchpath2']
];
$expected = [ 'prpsearchpath1', 'prpsearchpath2' ];
@$got = BSSched::BuildJob::expandkiwipath(@$fixture);
$expected = [ 'prpsearchpath1', 'prpsearchpath2' ];
is_deeply($got,$expected,'Checking with _obsrepositories and prpsearchpath');

################################################################################
$fixture = [
{},
{
path => [
{ path => 'abc' , project => '_obsrepositories' },
],
},
];
$expected = [ ];
@$got = BSSched::BuildJob::expandkiwipath(@$fixture);
@$got = BSSched::BuildJob::expandkiwipath({}, @$fixture);
$expected = [];
is_deeply($got,$expected,'Checking with _obsrepositories w/o prpsearchpath');
################################################################################
@$got = BSSched::BuildJob::expandkiwipath();
$expected = [ 'openSUSE:Factory/standard' ];
@$got = BSSched::BuildJob::expandkiwipath({});
is_deeply($got,[],'Checking empty $info->{path} element');

### Testing BSSched::BuildJob::jobname
Expand Down

0 comments on commit 914b475

Please sign in to comment.