From d1d60906a81f7e40b3960a21c87efe1c5b74aa7a Mon Sep 17 00:00:00 2001 From: Michael Schroeder Date: Fri, 22 Jun 2018 11:50:24 +0200 Subject: [PATCH 1/3] [backend] fix and refactor kiwimode detection The old code could not copy with a Dockerfile generated by a service. --- src/backend/bs_worker | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/src/backend/bs_worker b/src/backend/bs_worker index 47a1e84a9a5..4140a756070 100755 --- a/src/backend/bs_worker +++ b/src/backend/bs_worker @@ -2207,16 +2207,26 @@ sub getbinaries_buildenv { return (); # no meta } +sub getkiwimode { + my ($buildinfo) = @_; + my $file = $buildinfo->{'file'}; + $file =~ s/^_service:.*://; + my $kiwimode; + $kiwimode = 'image' if $file =~ /\.kiwi$/; + $kiwimode = 'product' if $kiwimode && $buildinfo->{'imagetype'} && $buildinfo->{'imagetype'}->[0] eq 'product'; + $kiwimode = 'docker' if $file eq 'Dockerfile'; + $kiwimode = 'fissile' if $file eq 'fissile.yml'; + return $kiwimode; +} + sub getbinaries { my ($buildinfo, $dir, $srcdir, $preinstallimagedata, $kiwiorigins) = @_; return getbinaries_buildenv($buildinfo, $dir, $srcdir) if $buildinfo->{'hasbuildenv'}; mkdir_p($dir); - my $kiwimode; - $kiwimode = 'image' if $buildinfo->{'file'} =~ /\.kiwi$/; - $kiwimode = 'docker' if $buildinfo->{'file'} eq 'Dockerfile'; - $kiwimode = 'fissile' if $buildinfo->{'file'} eq 'fissile.yml'; + my $recipefile = $buildinfo->{'file'}; + my $kiwimode = getkiwimode($buildinfo); # we need the Build package for queryhdrmd5 importbuild() unless defined &Build::queryhdrmd5; @@ -2790,11 +2800,7 @@ sub dobuild { my $repoid = $buildinfo->{'repository'}; my $arch = $buildinfo->{'arch'}; my $helperarch = $buildinfo->{'hostarch'} || $arch; - my $kiwimode; - $kiwimode = 'image' if $buildinfo->{'file'} =~ /\.kiwi$/; - $kiwimode = 'product' if $kiwimode && $buildinfo->{'imagetype'} && $buildinfo->{'imagetype'}->[0] eq 'product'; - $kiwimode = 'docker' if $buildinfo->{'file'} eq 'Dockerfile'; - $kiwimode = 'fissile' if $buildinfo->{'file'} eq 'fissile.yml'; + my $kiwimode = getkiwimode($buildinfo); my $kiwiorigins; my $stats = {}; my $deltamode; @@ -2881,11 +2887,7 @@ sub dobuild { $buildinfo->{'rootforbuild'} = 1 if $buildinfo->{'file'} =~ /\.kiwi$/; $buildinfo->{'file'} = $followupmode; # recalc kiwimode as we changed the file - undef $kiwimode; - $kiwimode = 'image' if $buildinfo->{'file'} =~ /\.kiwi$/; - $kiwimode = 'product' if $kiwimode && $buildinfo->{'imagetype'} && $buildinfo->{'imagetype'}->[0] eq 'product'; - $kiwimode = 'docker' if $buildinfo->{'file'} eq 'Dockerfile'; - $kiwimode = 'fissile' if $buildinfo->{'file'} eq 'fissile.yml'; + $kiwimode = getkiwimode($buildinfo); @meta = split("\n", readstr("$srcdir/meta")); } else { $kiwiorigins = {} if $kiwimode; # always for kiwi From 23129a9ab556a4ba01603a281dbea19a03a4b722 Mon Sep 17 00:00:00 2001 From: Michael Schroeder Date: Fri, 22 Jun 2018 11:54:14 +0200 Subject: [PATCH 2/3] [backend] fix handling of build time service deps They need to go into the sysdeps expansion so that they are available when the build script runs the services. --- src/backend/BSSched/BuildJob.pm | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/src/backend/BSSched/BuildJob.pm b/src/backend/BSSched/BuildJob.pm index fc609d372fa..df413557f90 100644 --- a/src/backend/BSSched/BuildJob.pm +++ b/src/backend/BSSched/BuildJob.pm @@ -875,9 +875,12 @@ sub create { # a new one. expand usedforbuild. write info file. my $buildtype = $pdata->{'buildtype'} || Build::recipe2buildtype($info->{'file'}); + if ($buildtype eq 'kiwi') { # split kiwi buildtype into kiwi-image and kiwi-product + $buildtype = $info->{'imagetype'} && $info->{'imagetype'}->[0] eq 'product' ? 'kiwi-product' : 'kiwi-image'; + } my $kiwimode; - $kiwimode = $buildtype if $buildtype eq 'kiwi' || $buildtype eq 'docker' || $buildtype eq 'fissile'; + $kiwimode = $buildtype if $buildtype eq 'kiwi-image' || $buildtype eq 'kiwi-product' || $buildtype eq 'docker' || $buildtype eq 'fissile'; my $syspath; my $searchpath = path2buildinfopath($gctx, $ctx->{'prpsearchpath'}); @@ -904,18 +907,20 @@ sub create { @btdeps = BSUtil::unify(@btdeps); } - # calculate sysdeps (cannot cache in the kiwi case) - my @sysdeps; - if ($buildtype eq 'kiwi') { - my $kiwitype = ''; - $kiwitype = $info->{'imagetype'} && $info->{'imagetype'}->[0] eq 'product' ? 'kiwi-product' : 'kiwi-image'; - @sysdeps = grep {/^kiwi-.*:/} @{$info->{'dep'} || []}; - @sysdeps = Build::get_sysbuild($bconf, $kiwitype, [ @sysdeps, @btdeps, @{$ctx->{'extradeps'} || []} ]); + # calculate sysdeps + my @sysdeps = @btdeps; + if ($buildtype eq 'kiwi-image' || $buildtype eq 'kiwi-product') { + unshift @sysdeps, grep {/^kiwi-.*:/} @{$info->{'dep'} || []}; + push @sysdeps, @{$ctx->{'extradeps'} || []}; + } + if (@sysdeps) { + @sysdeps = Build::get_sysbuild($bconf, $buildtype, [ @sysdeps ]); # cannot cache... } else { $ctx->{"sysbuild_$buildtype"} ||= [ Build::get_sysbuild($bconf, $buildtype) ]; @sysdeps = @{$ctx->{"sysbuild_$buildtype"}}; } add_expanddebug($ctx,'sysdeps expansion') if $expanddebug && @sysdeps; + @btdeps = () if @sysdeps; # already included in sysdeps # calculate packages needed for building my @bdeps = grep {!/^\// || $bconf->{'fileprovides'}->{$_}} @{$info->{'prereq'} || []}; From 25acebcc0e62b121b2b455a0b6fdb62bc2193b0e Mon Sep 17 00:00:00 2001 From: Michael Schroeder Date: Fri, 22 Jun 2018 14:18:52 +0200 Subject: [PATCH 3/3] [backend] implement linkedbuild exclusion logic in wipeobsolete We still need to implement 'localdep' wiping, but this is a first step... --- src/backend/BSSched/Checker.pm | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/backend/BSSched/Checker.pm b/src/backend/BSSched/Checker.pm index 84d99a77c98..18daa85c206 100644 --- a/src/backend/BSSched/Checker.pm +++ b/src/backend/BSSched/Checker.pm @@ -260,6 +260,8 @@ sub wipeobsolete { my $repoid = $ctx->{'repository'}; my $projpacks = $gctx->{'projpacks'}; my $myarch = $gctx->{'arch'}; + my $repo = (grep {$_->{'name'} eq $repoid} @{$projpacks->{$projid}->{'repository'} || []})[0]; + my $linkedbuild = $repo ? $repo->{'linkedbuild'} : 'all'; my $pdatas = $projpacks->{$projid}->{'package'} || {}; my $dstcache = { 'fullcache' => {}, 'bininfocache' => {} }; my $hadobsolete; @@ -275,6 +277,10 @@ sub wipeobsolete { if (($pdata->{'error'} || '') eq 'excluded') { $reason = 'excluded'; } else { + if (exists($pdata->{'originproject'})) { + # package from project link + $reason = 'excluded' if !$linkedbuild || ($linkedbuild ne 'localdep' && $linkedbuild ne 'all'); + } my %info = map {$_->{'repository'} => $_} @{$pdata->{'info'} || []}; my $info = $info{$repoid}; $reason = 'excluded' if $info && ($info->{'error'} || '') eq 'excluded';