Skip to content

Commit

Permalink
[backend] split %cando into a separate module, first try at supportin…
Browse files Browse the repository at this point in the history 
…g 'hostarch' in buildconfig
  • Loading branch information
@mlschroe
mlschroe committed Feb 9, 2012
1 parent 0751fdd commit 13c161b
Show file tree
Hide file tree
Showing 6 changed files with 150 additions and 56 deletions.
77 changes: 77 additions & 0 deletions src/backend/BSCando.pm
View file
@@ -0,0 +1,77 @@
#!/usr/bin/perl -w
#
# Copyright (c) 2006-2012 Novell Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program (see the file COPYING); if not, write to the
# Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
#
################################################################
#
# Defines what architectures we can build
#

package BSCando;

#
# the cando table mapps the host architecture to the repository architectures
# that can be built on the host.
#

#FIXME 3.0: obsolete the not exiting arm architectures

our %cando = (
'armv4l' => [ 'armv4l' ],
'armv5l' => [ 'armv4l', 'armv5l' , 'armv5el' ],
'armv6l' => [ 'armv4l', 'armv5l', 'armv6l' , 'armv5el', 'armv6el' ],
'armv7l' => [ 'armv4l', 'armv5l', 'armv6l', 'armv7l', 'armv5el', 'armv6el', 'armv7el', 'armv7hl', 'armv8el' ],

'sh4' => [ 'sh4' ],

'i586' => [ 'i586' ],
'i686' => [ 'i586', 'i686' ],
'x86_64' => [ 'x86_64', 'i586:linux32', 'i686:linux32' ],

'parisc' => [ 'hppa', 'hppa64:linux64' ],
'parisc64'=> [ 'hppa64', 'hppa:linux32' ],

'ppc' => [ 'ppc' ],
'ppc64' => [ 'ppc64', 'ppc:powerpc32' ],

'ia64' => [ 'ia64' ],

's390' => [ 's390' ],
's390x' => [ 's390x', 's390:s390' ],

'sparc' => [ 'sparcv8', 'sparc' ],
'sparc64' => [ 'sparc64v', 'sparc64', 'sparcv9v', 'sparcv9', 'sparcv8:linux32', 'sparc:linux32' ],

'mips' => [ 'mips' ],
'mips64' => [ 'mips64', 'mips:mips32' ],

'local' => [ 'local' ],
);

our %knownarch;

for my $harch (keys %cando) {
for my $arch (@{$cando{$harch} || []}) {
if ($arch =~ /^(.*):/) {
$knownarch{$1}->{$harch} = $arch;
} else {
$knownarch{$arch}->{$harch} = $arch;
}
}
}

1;
65 changes: 38 additions & 27 deletions src/backend/bs_dispatch
View file
Expand Up @@ -47,6 +47,7 @@ use BSConfig;
use BSRPC;
use BSUtil;
use BSXML;
use BSCando;

use strict;

Expand All @@ -71,30 +72,11 @@ BSUtil::drop_privs_to($BSConfig::bsuser, $BSConfig::bsgroup);
my $port = 5252; #'RR'
$port = $1 if $BSConfig::reposerver =~ /:(\d+)$/;

my %cando = (
# this is breaking native builds of arm/mips/sh4, the cross compile compat needs to become configurable per prpa in future
'i586' => [ 'i586', 'armv4l', 'armv5l', 'armv6l', 'armv7l', 'armv5el', 'armv6el', 'armv7el', 'armv7hl', 'armv8el', 'mips', 'mipsel', 'sh4'],
'i686' => [ 'i586', 'i686', 'armv4l', 'armv5l', 'armv6l', 'armv7l', 'armv5el', 'armv6el', 'armv7el', 'armv7hl', 'armv8el', 'mips', 'mipsel', 'sh4'],
'x86_64' => ['x86_64', 'i586', 'i686', 'armv4l', 'armv5l', 'armv6l', 'armv7l', 'armv5el', 'armv6el', 'armv7el', 'armv7hl', 'armv8el', 'mips', 'mipsel', 'sh4'],
#
'ppc' => [ 'ppc' ],
'ppc64' => [ 'ppc', 'ppc64', ],
'armv4l' => [ 'armv4l' ],
'armv5l' => [ 'armv4l', 'armv5l', 'armv5el', ],
'armv6l' => [ 'armv4l', 'armv5l', 'armv6l', 'armv5el', 'armv6el' ],
'armv7l' => [ 'armv4l', 'armv5l', 'armv6l', 'armv7l', 'armv5el', 'armv6el', 'armv7el' ],
'sh4' => [ 'sh4'],
'parisc' => ['hppa', 'hppa64:linux64'],
'parisc64'=> ['hppa64', 'hppa:linux32'],
'ia64' => ['ia64'],
's390' => ['s390'],
's390x' => ['s390x', 's390'],
'sparc' => ['sparcv8', 'sparc'],
'sparc64' => ['sparc64v', 'sparc64', 'sparcv9v', 'sparcv9', 'sparcv8:linux32' , 'sparc:linux32'],
'mips' => ['mips'],
'mips64' => ['mips64', 'mips'],
'local' => ['local'],
);
# strip helpers from cando
my %cando = %BSCando::cando;
for my $hostarch (values %cando) {
s/:.*// for @{$hostarch || []};
}

# 4h build will add .5 to the load
# 4h idle will half the load
Expand Down Expand Up @@ -371,9 +353,35 @@ while (1) {
#print "finding jobs\n";
my %jobs;
my %maybesrcchange;
for my $arch (sort keys %idlearch) {
my @archs = sort keys %idlearch;
my %archdone;
my %crossarchlist;
while (@archs) {
my $arch = shift @archs;
next if $archdone{$arch};
$archdone{$arch} = 1;
my $ic = $infocache{$arch} || {};
my @b = grep {!/^\./} ls("$jobsdir/$arch");
my @crossb = grep {/:cross$/} @b;
if (@crossb) {
my %crossarchs;
for (@crossb) {
push @{$crossarchs{$2}}, $1 if /^(.*):([^:]+):cross$/;
}
for my $crossarch (sort keys %crossarchs) {
next unless $idlearch{$crossarch};
my %cj = map {$_ => 1} ls("$jobsdir/$crossarch");
# deltete orphaned marker
for (@{$crossarchs{$crossarch}}) {
next if $cj{$_};
print " - deleting orphaned cross marker $arch/$_:${crossarch}:cross\n";
unlink("$jobsdir/$arch/$_:${crossarch}:cross");
}
push @archs, $crossarch;
$crossarchlist{$crossarch}->{$arch} = 1;
}
@b = grep {!/:cross$/} @b;
}
my %locked = map {$_ => 1} grep {/:status$/} @b;
my %notlocked = map {$_ => 1} grep {!$locked{$_}} @b;
for (grep {!$notlocked{$_}} keys (%{$infocache{$arch} || {}})) {
Expand Down Expand Up @@ -496,7 +504,10 @@ while (1) {
while (@jobprpas) {
my $prpa = shift @jobprpas;
my $arch = (split('/', $prpa))[2];
next unless @{$idlearch{$arch} || []};
if (!@{$idlearch{$arch} || []}) {
next unless $crossarchlist{$arch}; # where can be also build that?
next unless grep {@{$idlearch{$_} || []}} keys %{$crossarchlist{$arch}};
}
my @b = @{$jobs{$prpa} || []};
next unless @b;

Expand Down Expand Up @@ -563,7 +574,7 @@ while (1) {
last;
}
}
my @idle = List::Util::shuffle(@{$idlearch{$arch} || []});
my @idle = List::Util::shuffle(@{$idlearch{$ic->{'hostarch'} || $arch} || []});
last unless @idle;
my %poweridle;
if ($powerjobs{$job}) {
Expand Down
2 changes: 1 addition & 1 deletion src/backend/bs_repserver
View file
Expand Up @@ -1148,7 +1148,7 @@ sub workerstatus {
@a = ls($jobsdir) unless $cgi->{'scheduleronly'};
for my $a (@a) {
next unless -d "$jobsdir/$a";
my @d = grep {!/^\./ && !/:(?:dir|new)$/} ls("$jobsdir/$a");
my @d = grep {!/^\./ && !/:(?:dir|new|cross)$/} ls("$jobsdir/$a");
my %d = map {$_ => 1} @d;
for my $d (grep {/:status$/} @d) {
delete $d{$d};
Expand Down
31 changes: 29 additions & 2 deletions src/backend/bs_sched
View file
Expand Up @@ -48,6 +48,7 @@ use Build;
use BSDB;
use Meta;
use BSSolv;
use BSCando;

use strict;

Expand Down Expand Up @@ -91,6 +92,10 @@ if (@ARGV && ($ARGV[0] eq '--exit' || $ARGV[0] eq '--stop')) {

my $myarch = $ARGV[0] || 'i586';

if (!$BSCando::knownarch{$myarch}) {
die("Architecture '$myarch' is unknown, please adapt BSCando.pm\n");
}

my $myjobsdir = "$jobsdir/$myarch";
my $myeventdir = "$eventdir/$myarch";

Expand Down Expand Up @@ -506,6 +511,17 @@ sub killbuilding {
}
}

sub add_crossmarker {
my ($bconf, $job) = @_;
my $hostarch = $bconf->{'hostarch'};
return if $hostarch eq $myarch;
return unless $BSCando::knownarch{$hostarch};
my $marker = "$jobsdir/$hostarch/$job:$hostarch:cross";
return if -e $marker;
mkdir_p("$jobsdir/$hostarch");
BSUtil::touch($marker);
}

#
# set_building - create a new build job
#
Expand Down Expand Up @@ -536,8 +552,11 @@ sub set_building {
my $prp = "$projid/$repoid";
my $srcmd5 = $pdata->{'srcmd5'};
my $job = jobname($prp, $packid);
return "$job-$srcmd5" if -s "$myjobsdir/$job-$srcmd5";
return $job if -s "$myjobsdir/$job";
if (-s "$myjobsdir/$job-$srcmd5") {
add_crossmarker($bconf, "$job-$srcmd5") if $bconf->{'hostarch'};
return "$job-$srcmd5";
}
return $job if -s "$myjobsdir/$job"; # obsolete
my @otherjobs = grep {/^\Q$job\E-[0-9a-f]{32}$/} ls($myjobsdir);
$job = "$job-$srcmd5";

Expand Down Expand Up @@ -695,6 +714,7 @@ sub set_building {
$binfo->{'debuginfo'} = 1 if $debuginfo;

writexml("$myjobsdir/.$job", "$myjobsdir/$job", $binfo, $BSXML::buildinfo);
add_crossmarker($bconf, $job) if $bconf->{'hostarch'};
# all done. the dispatcher will now pick up the job and send it
# to a worker.
$ourjobs{$job} = 1;
Expand Down Expand Up @@ -5256,6 +5276,12 @@ while(1) {
$prpfinished{$prp} = 1;
next;
}
if ($bconf->{'hostarch'} && !$BSCando::knownarch{$bconf->{'hostarch'}}) {
print " - $prp: bad hostarch ($bconf->{'hostarch'})\n";
set_repo_state($prp, 'broken', "bad hostarch ($bconf->{'hostarch'})");
$prpfinished{$prp} = 1;
next;
}
print " - $prp\n";

my $repo = (grep {$_->{'name'} eq $repoid} @{$projpacks->{$projid}->{'repository'} || []})[0];
Expand Down Expand Up @@ -5811,6 +5837,7 @@ while(1) {
if (-s "$myjobsdir/$job") {
# print " - $packid ($packtype)\n";
# print " already scheduled\n";
add_crossmarker($job) if $bconf->{'hostarch'};
my $useforbuildenabled = enabled($repoid, $pdata->{'useforbuild'}, $prjuseforbuildenabled);
$building{$packid} = $job;
$notready{$pname} = 1 if $useforbuildenabled;
Expand Down
30 changes: 4 additions & 26 deletions src/backend/bs_worker
View file
Expand Up @@ -46,6 +46,7 @@ use BSXML;
use BSKiwiXML;
use BSHTTP;
use BSBuild;
use BSCando;

use strict;

Expand Down Expand Up @@ -83,29 +84,6 @@ my $buildlog_maxidle = 8 * 3600;
my $xenstore_maxsize = 20 * 1000000;
my $gettimeout = 3600; # 1 hour timeout to avoid forever hanging workers

my %cando = (
'armv4l' => ['armv4l' ],
'armv5l' => ['armv4l', 'armv5l' , 'armv5el' ],
'armv6l' => ['armv4l', 'armv5l', 'armv6l' , 'armv5el', 'armv6el' ],
'armv7l' => ['armv4l', 'armv5l', 'armv6l', 'armv7l', 'armv5el', 'armv6el', 'armv7el', 'armv7hl', 'armv8el' ],
'sh4' => ['sh4' ],
'i586' => [ 'i586', ],
'i686' => [ 'i586', 'i686', ],
'x86_64' => ['x86_64', 'i586:linux32', 'i686:linux32'],
#
'parisc' => ['hppa', 'hppa64:linux64'],
'parisc64'=> ['hppa64', 'hppa:linux32'],
'ppc' => ['ppc'],
'ppc64' => ['ppc64', 'ppc:powerpc32'],
'ia64' => ['ia64'],
's390' => ['s390'],
's390x' => ['s390x', 's390:s390'],
'sparc' => ['sparcv8', 'sparc'],
'sparc64' => ['sparc64v', 'sparc64', 'sparcv9v', 'sparcv9', 'sparcv8:linux32', 'sparc:linux32'],
'mips' => ['mips'],
'mips64' => ['mips64', 'mips:mips32'],
);

$hostcheck = $BSConfig::workerhostcheck if defined($BSConfig::workerhostcheck);

sub lockstate {
Expand Down Expand Up @@ -179,7 +157,7 @@ Usage: $0 [OPTION] --root <directory> --statedir <directory>
--arch : define hostarch (overrides 'uname -m')
currently supported architectures:
@{[sort keys %cando]}
@{[sort keys %BSCando::cando]}
--kvm : enable kvm
Expand Down Expand Up @@ -469,7 +447,7 @@ if (!$hostarch) {
die("could not determine hostarch\n") unless $hostarch;
}

die("arch $hostarch cannot build anything!\n") unless $cando{$hostarch} || ($hostarch eq 'local' && $localkiwi);
die("arch $hostarch cannot build anything!\n") unless $BSCando::cando{$hostarch} || ($hostarch eq 'local' && $localkiwi);

$srcserver = $BSConfig::srcserver unless defined $srcserver;
@reposervers = @BSConfig::reposervers unless @reposervers;
Expand Down Expand Up @@ -1447,7 +1425,7 @@ sub dobuild {
$deltamode = 1 if $buildinfo->{'file'} eq '_delta';

my $helper = '';
/^\Q$helperarch\E:(.*)$/ && ($helper = $1) for @{$cando{$hostarch}};
/^\Q$helperarch\E:(.*)$/ && ($helper = $1) for @{$BSCando::cando{$hostarch}};

my @lt = localtime(time());
my $timestring = sprintf "%04d-%02d-%02d %02d:%02d:%02d", $lt[5] + 1900, $lt[4] + 1, @lt[3,2,1,0];
Expand Down
1 change: 1 addition & 0 deletions src/backend/worker/BSCando.pm
View file
@@ -0,0 +1 @@
../BSCando.pm

0 comments on commit 13c161b

Please sign in to comment.