Skip to content

Commit

Permalink
Merge pull request #8764 from mlschroe/master
Browse files Browse the repository at this point in the history 
[backend] implement automatic ptf package generation
  • Loading branch information
@adrianschroeter
adrianschroeter committed Nov 22, 2019
2 parents dfe65b9 + 64c8b78 commit c889538
Show file tree
Hide file tree
Showing 6 changed files with 138 additions and 54 deletions.
1 change: 1 addition & 0 deletions dist/obs-server.spec
View file
Original file line number Diff line number Diff line change
Expand Up @@ -709,6 +709,7 @@ fi
/usr/lib/obs/server/bs_notifyforward
/usr/lib/obs/server/worker
/usr/lib/obs/server/worker-deltagen.spec
/usr/lib/obs/server/obs-ptf.spec
%config(noreplace) /usr/lib/obs/server/BSConfig.pm
%config(noreplace) /etc/slp.reg.d/*
# created via %%post, since rpm fails otherwise while switching from
Expand Down
4 changes: 4 additions & 0 deletions src/backend/BSSched/BuildJob.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -801,6 +801,10 @@ sub create_jobdata {
$binfo->{'module'} = $bconf->{'modules'} if $bconf->{'modules'};
my $obsname = $gctx->{'obsname'};
$binfo->{'disturl'} = "obs://$obsname/$projid/$repoid/$pdata->{'srcmd5'}-$packid" if defined($obsname) && defined($packid);

# no release/debuginfo for patchinfo and deltarpm builds
return $binfo if $buildtype eq 'patchinfo' || $buildtype eq 'deltarpm';

if (defined($packid) && exists($pdata->{'versrel'})) {
$binfo->{'versrel'} = $pdata->{'versrel'};
# find the last build count we used for this version/release
Expand Down
85 changes: 62 additions & 23 deletions src/backend/BSSched/BuildJob/Patchinfo.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -206,9 +206,6 @@ sub check {

return ('broken', $broken) if $broken;

my @new_meta;
push @new_meta, ($pdata->{'verifymd5'} || $pdata->{'srcmd5'})." $packid";

if ($ptype eq 'local') {
# only rebuild if patchinfo source changes
my @meta;
Expand All @@ -217,7 +214,8 @@ sub check {
close F;
chomp @meta;
}
if (@meta == 1 && $meta[0] eq $new_meta[0]) {
my $verifymd5 = $pdata->{'verifymd5'} || $pdata->{'srcmd5'};
if (@meta == 1 && $meta[0] eq "$verifymd5 $packid") {
print " - $packid (patchinfo)\n";
print " nothing changed\n";
return ('done');
Expand Down Expand Up @@ -326,25 +324,10 @@ sub check {

return ('broken', 'no binaries found') unless @tocopy;

for (sort(keys %metas)) {
push @new_meta, "$metas{$_} $_";
}

# compare with stored meta
my @meta;
if (open(F, '<', "$gdst/:meta/$packid")) {
@meta = <F>;
close F;
chomp @meta;
}
if (@meta == @new_meta && join("\n", @meta) eq join("\n", @new_meta)) {
print " - $packid (patchinfo)\n";
print " nothing changed\n";
return ('done');
}

# now collect...
return ('scheduled', [ \@tocopy, \%metas, $ptype]);
# compare against old meta and build if it does not match
my @new_meta;
push @new_meta, "$metas{$_} $_" for sort keys %metas;
return BSSched::BuildJob::metacheck($ctx, $packid, $pdata, 'patchinfo', \@new_meta, [ \@tocopy, \%metas, $ptype ]);
}


Expand All @@ -354,6 +337,56 @@ sub check {
=cut

sub build_ptf_job {
my ($self, $ctx, $packid, $pdata, $job, $metas, $reason) = @_;

my $gctx = $ctx->{'gctx'};
my $myjobsdir = $gctx->{'myjobsdir'};
my $obssrcdir = $gctx->{'obssrcdir'};
my $jobdatadir = "$myjobsdir/$job:dir";
# get data for all rpms
my @rpms;
for my $bin (sort(grep {/^[^\.].*\.rpm$/s} ls($jobdatadir))) {
eval {
my $d = Build::query("$jobdatadir/$bin", 'evra' => 1, 'description' => 1);
$d->{'evr'} = "$d->{'version'}-$d->{'release'}";
$d->{'evr'} = "$d->{'epoch'}:$d->{'evr'}" if $d->{'epoch'};
push @rpms, $d;
};
if ($@) {
my $err = "$bin: $@";
chomp $err;
return ('broken', $err);
}
}
return ('broken', 'no rpms') unless @rpms;
# write a ptf.spec file
my $patchinfo = { %{$pdata->{'patchinfo'}} }; # copy so we can modify
return ('broken', 'no incident number') unless $patchinfo->{'incident'};
$patchinfo->{'version'} ||= 1;
$patchinfo->{'description'} =~ s/\n+$/\n/s if $patchinfo->{'description'};
my @ptfspec = split("\n", readstr("$obssrcdir/obs-ptf.spec"));
for my $ptfline (splice @ptfspec) {
$ptfline =~ s/\@patchinfo-(.*?)\@/$patchinfo->{$1}/ge;
if ($ptfline =~ /\@rpm-.*?\@/) {
for my $rpm (@rpms) {
my $l = $ptfline;
$l =~ s/\@rpm-(.*?)\@/$rpm->{$1}/ge;
push @ptfspec, $l;
}
} else {
push @ptfspec, $ptfline;
}
}
writestr("$jobdatadir/ptf.spec", undef, join("\n", @ptfspec)."\n");
my @new_meta = ($pdata->{'verifymd5'} || $pdata->{'srcmd5'})." $packid";
push @new_meta, "$metas->{$_} $_" for sort keys %$metas;
writestr("$jobdatadir/meta", undef, join("\n", @new_meta)."\n");
my $pdata_job = { 'srcmd5' => $pdata->{'srcmd5'}, 'buildtype' => 'patchinfo' };
my $info_job = { 'file' => '_ptf' };
return BSSched::BuildJob::create($ctx, $packid, $pdata_job, $info_job, [], [], $reason, 0);
}

sub build {
my ($self, $ctx, $packid, $pdata, $info, $data) = @_;

Expand All @@ -365,6 +398,7 @@ sub build {
my @tocopy = @{$data->[0]};
my $ckmetas = $data->[1];
my $ptype = $data->[2];
my $reason = $data->[3];
my $reporoot = $gctx->{'reporoot'};

print " - $packid (patchinfo)\n";
Expand Down Expand Up @@ -556,6 +590,11 @@ sub build {

$broken ||= 'no binaries found' unless @upackages;

if (($patchinfo->{'category'} || '') eq 'ptf') {
# create a ptf build job
return build_ptf_job($self, $ctx, $packid, $pdata, $job, \%metas, $reason);
}

my $update = {};
$update->{'status'} = $patchinfo->{'retracted'} ? 'retracted' : 'stable';
$update->{'from'} = $patchinfo->{'packager'} if $patchinfo->{'packager'};
Expand Down
1 change: 1 addition & 0 deletions src/backend/bs_sched
View file
Original file line number Diff line number Diff line change
Expand Up @@ -417,6 +417,7 @@ my $gctx = {
'remoteproxy' => $BSConfig::proxy,
'asyncmode' => $asyncmode,
'critlogfile' => "$_logdir/scheduler_${_myarch}.crit.log",
'obssrcdir' => $INC[0],

# repository state cache
# 'lastscan' last time we scanned
Expand Down
66 changes: 35 additions & 31 deletions src/backend/bs_worker
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1048,30 +1048,23 @@ sub getsources {
return @meta;
}

sub getdeltasources {
sub getjobdata {
my ($buildinfo, $dir) = @_;

my @meta;
push @meta, ($buildinfo->{'verifymd5'} || $buildinfo->{'srcmd5'})." $buildinfo->{'package'}";
my $server = $buildinfo->{'reposerver'};
my $res = BSRPC::rpc({
'uri' => "$server/getjobdata",
'directory' => $dir,
'timeout' => $gettimeout,
'receiver' => \&BSHTTP::cpio_receiver,
}, undef, "job=$buildinfo->{'job'}", "arch=$buildinfo->{'arch'}", "jobid=$buildinfo->{'jobid'}");
return @meta;
return $res;
}

sub getfollowupsources {
my ($buildinfo, $dir) = @_;
my $server = $buildinfo->{'reposerver'};
my $res = BSRPC::rpc({
'uri' => "$server/getjobdata",
'directory' => $dir,
'timeout' => $gettimeout,
'receiver' => \&BSHTTP::cpio_receiver,
}, undef, "job=$buildinfo->{'job'}", "arch=$buildinfo->{'arch'}", "jobid=$buildinfo->{'jobid'}");
my $res = getjobdata($buildinfo, $dir);
die("no old sources in followup job\n") unless @$res;
# need to get buildenv from old sources ;(
$server = $buildinfo->{'srcserver'} || $srcserver;
Expand Down Expand Up @@ -2925,8 +2918,8 @@ sub dobuild {
my $kiwimode = getkiwimode($buildinfo);
my $kiwiorigins;
my $stats = {};
my $deltamode;
$deltamode = 1 if $buildinfo->{'file'} eq '_delta';
my $deltamode = $buildinfo->{'file'} eq '_delta' ? 1 : undef;
my $ptfmode = $buildinfo->{'file'} eq '_ptf' ? 1 : undef;
my $followupmode = $buildinfo->{'followupfile'};
my $followupcopy = {};
my $needobspackage;
Expand Down Expand Up @@ -2958,7 +2951,7 @@ sub dobuild {
rm_rf($oldpkgdir) if -d $oldpkgdir;

# create buildenv data
if (!$kiwimode && !$followupmode && !$deltamode) {
if (!$kiwimode && !$followupmode && !$deltamode && !$ptfmode) {
$buildinfo->{'outbuildinfo'} = {
'project' => $projid,
'package' => $packid,
Expand All @@ -2983,13 +2976,22 @@ sub dobuild {
print "fetching sources, ";
mkdir($srcdir) || die("mkdir $srcdir: $!\n");
if ($deltamode) {
push @meta, getdeltasources($buildinfo, $srcdir);
push @meta, ($buildinfo->{'verifymd5'} || $buildinfo->{'srcmd5'})." $buildinfo->{'package'}";
getjobdata($buildinfo, $srcdir);
print "packages, ";
getbinaries($buildinfo, $pkgdir, $srcdir, $preinstallimagedata);
undef $oldpkgdir;
if (! -s "$srcdir/deltagen.spec") {
my $spec = readstr("$statedir/worker/worker-deltagen.spec", 1) || readstr("worker-deltagen.spec");
writestr("$srcdir/deltagen.spec", undef, $spec);
}
$buildinfo->{'file'} = 'deltagen.spec';
} elsif ($ptfmode) {
getjobdata($buildinfo, $srcdir);
print "packages, ";
getbinaries($buildinfo, $pkgdir, $srcdir, $preinstallimagedata);
undef $oldpkgdir;
my $spec = readstr("$statedir/worker/worker-deltagen.spec", 1) || readstr("worker-deltagen.spec");
writestr("$srcdir/worker-deltagen.spec", undef, $spec);
$buildinfo->{'file'} = 'worker-deltagen.spec';
$buildinfo->{'file'} = 'ptf.spec';
} elsif ($followupmode) {
getfollowupsources($buildinfo, $srcdir);
getsslcert($buildinfo, $srcdir);
Expand Down Expand Up @@ -3076,20 +3078,22 @@ sub dobuild {

if ($deltamode) {
# replace @mopt@ in specfile
my $spec = readstr("$srcdir/worker-deltagen.spec");
my $mopt;
$mopt = $BSConfig::deltagen_mopt if defined $BSConfig::deltagen_mopt;
importbuild() unless defined &Build::queryhdrmd5;
eval {
my $bconf = Build::read_config($buildinfo->{'arch'}, [ split("\n", $config) ]);
for (@{$bconf->{'repotype'}}) {
$mopt = $1 eq '' ? undef : $1 if /^deltagen-mopt:(\d*)/;
}
};
$mopt = 512 unless defined $mopt;
$mopt = '' unless $mopt;
$spec =~ s/\@mopt\@/$mopt/sg;
writestr("$srcdir/worker-deltagen.spec", undef, $spec);
my $spec = readstr("$srcdir/deltagen.spec");
if ($spec =~ /\@mopt\@/s) {
my $mopt;
$mopt = $BSConfig::deltagen_mopt if defined $BSConfig::deltagen_mopt;
importbuild() unless defined &Build::queryhdrmd5;
eval {
my $bconf = Build::read_config($buildinfo->{'arch'}, [ split("\n", $config) ]);
for (@{$bconf->{'repotype'}}) {
$mopt = $1 eq '' ? undef : $1 if /^deltagen-mopt:(\d*)/;
}
};
$mopt = 512 unless defined $mopt;
$mopt = '' unless $mopt;
$spec =~ s/\@mopt\@/$mopt/sg;
writestr("$srcdir/deltagen.spec", undef, $spec);
}
}

my $release = $buildinfo->{'release'};
Expand Down
35 changes: 35 additions & 0 deletions src/backend/obs-ptf.spec
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
Name: ptf-@patchinfo-incident@
Version: @patchinfo-version@
Release: 0
Summary: @patchinfo-summary@
License: SUSE-Redistributable-Content
Group: System/Packages
BuildRoot: %{_tmppath}/%{name}-%{version}-build
Provides: ptf() = @patchinfo-incident@-@patchinfo-version@
Requires: (@rpm-name@ = @rpm-evr@ if @rpm-name@)

%description
@patchinfo-description@

This ptf contains the following packages:
@rpm-name@-@rpm-version@-@rpm-release@.@rpm-arch@

%build
cd %_sourcedir
odir="%_topdir/OTHER"
cp *.rpm "$odir"
mkdir -p %{buildroot}/%{_defaultdocdir}/%{name}
cat >%{buildroot}/%{_defaultdocdir}/%{name}/README <<'EOF'
@patchinfo-description@
This ptf contains the following packages:
@rpm-name@-@rpm-version@-@rpm-release@.@rpm-arch@
EOF

%files
%defattr(-,root,root)
%doc %{_defaultdocdir}/%{name}

%changelog

0 comments on commit c889538

Please sign in to comment.