bs_worker

#!/usr/bin/perl -w
#
# Copyright (c) 2006-2009 Michael Schroeder, Novell Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program (see the file COPYING); if not, write to the
# Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
#
################################################################
#
# Worker build process. Builds jobs received from a Repository Server,
# sends build binary packages back.
#

BEGIN {
  my ($wd) = $0 =~ m-(.*)/- ;
  $wd ||= '.';
  unshift @INC,  "$wd/build";
  unshift @INC,  "$wd";
}

use Digest::MD5 ();
use XML::Structured ':bytes';
use Data::Dumper;
use POSIX;
use Fcntl qw(:DEFAULT :flock);
BEGIN { Fcntl->import(':seek') unless defined &SEEK_SET; }

use Storable;

use BSRPC;
use BSServer;
use BSConfig;
use BSUtil;
use BSXML;
use BSKiwiXML;
use BSHTTP;
use BSBuild;
use BSCando;

use strict;

my @binsufs = qw{rpm deb pkg.tar.gz pkg.tar.xz};
my $binsufsre = join('|', map {"\Q$_\E"} @binsufs);

my $buildroot;
my $port;
my $statedir;
my $hostarch;
my $vm = '';
my $vm_tmpfs_mode;
my $vm_root = '';
my $vm_swap = '';
my $vm_kernel;
my $vm_initrd;
my $vm_memory;
my $vmdisk_rootsize;
my $vmdisk_swapsize;
my $vmdisk_filesystem;
my $vmdisk_mount_options;
my $hugetlbfs;
my $workerid;
my $srcserver;
my @reposervers;
my $testmode;
my $noworkercheck;
my $nobuildcodecheck;
my $oneshot;
my $silent;
my $hostcheck;
my $localkiwi;

my $jobs;
my $cachedir;
my $cachesize;

my $buildlog_maxsize = 500 * 1000000;
my $buildlog_maxidle = 8 * 3600;
my $xenstore_maxsize = 20 * 1000000;
my $gettimeout = 3600; # 1 hour timeout to avoid forever hanging workers

$hostcheck = $BSConfig::workerhostcheck if defined($BSConfig::workerhostcheck);

sub lockstate {
  while (1) {
    open(STATELOCK, '>>', "$statedir/state") || die("$statedir/state: $!\n");
    flock(STATELOCK, LOCK_EX) || die("flock $statedir/state: $!\n");
    my @s = stat(STATELOCK);
    last if $s[3];	# check nlink
    close(STATELOCK);	# race, try again
  }
  my $oldstate = readxml("$statedir/state", $BSXML::workerstate, 1);
  $oldstate = {} unless $oldstate;
  return $oldstate;
}

sub unlockstate {
  close(STATELOCK);
}

sub commitstate {
  my ($newstate) = @_;
  writexml("$statedir/state.new", "$statedir/state", $newstate, $BSXML::workerstate) if $newstate;
  close(STATELOCK);
}

sub trunc_logfile {
  my $lf = shift;
  open(LF, "<$lf") || return; 
  my $buf;
  sysread(LF, $buf, 1000000);
  $buf .= "\n\n[truncated]\n\n";
  sysseek(LF, -1000000, 2);
  sysread(LF, $buf, 1000000, length($buf));
  close LF;
  $buf .= "\nLogfile got too big, killed job.\n";
  open(LF, ">$lf.new") || return; 
  syswrite(LF, $buf);
  close LF;
  rename("$lf.new", $lf);
}

sub cleanup_job {
  if ($vm_tmpfs_mode) {
    qsystem("umount", $buildroot) && die("umount tmpfs failed: $!\n");
  }
}

sub kill_job {
  if (system("$statedir/build/build", "--root", $buildroot, ($vm ? ($vm, "$vm_root") : ()), "--kill")) {
    return 0;
  }
  cleanup_job();
  return 1;
}

sub usage {
  my ($ret) = @_;

print <<EOF;
Usage: $0 [OPTION] --root <directory> --statedir <directory>

       --root      : buildroot directory

       --port      : fixed port number

       --statedir  : state directory

       --id        : worker id

       --reposerver: define reposerver, can be used multiple times

       --arch      : define hostarch (overrides 'uname -m')
                     currently supported architectures: 
                     @{[sort keys %BSCando::cando]}

       --kvm       : enable kvm

       --xen       : enable xen

       --lxc       : enable lxc

       --tmpfs     : uses tmpfs (memory) for for the build root

       --device    : set kvm or xen root device (default is <root>/root file)

       --swap      : set kvm or xen swap device (default is <root>/swap file)

       --vm-kernel : set kernel to use (xen/kvm)

       --vm-initrd : set initrd to use (xen/kvm)

       --vm-memory : set amount of memory to use (xen/kvm)

       --vmdisk-rootsize <size>
                   : size of the root disk image (default 4096M)

       --vmdisk-swapsize <size>
                   : size of the swap disk image (default 1024M)

       --vmdisk-filesystem <none|ext3|ext4>
                   : filesystem to use for autosetup root disk image

       --hugetlbfs HUGETLBFSPATH
                   : Use hugetlb for memory management, path to mounted hugetlbfs.

       --test      : enable test mode

       --build     : just build the package, don't send anything back
                     (needs a buildinfo file as argument)

       --noworkerupdate
                   : do not check if the worker is up-to-date

       --nobuildupdate
                   : do not check if the build code is up-to-date

       --nocodeupdate
                   : do not update both worker and build code

       --jobs <nr> : hand over the number of parallel jobs to build

       --oneshot <seconds>
                   : just build one package, do not wait more then
                     <seconds> seconds if nothing is available

       --hostcheck <hostcheck>
                   : call to check if the host can build the package

       --cachedir <cachedir>
       --cachesize <size_in_mb>
                   : use cachedir to cache fetched binaries

       --help      : this message

EOF
  exit $ret || 0;
}

my @saveargv = @ARGV;	# so we can restart ourself
my $justbuild;
my $exitrestart;

my $exitrestart_timeout = 300;	# wait max 5 minuntes

exit(0) if @ARGV == 1 && $ARGV[0] eq '--selftest';

while (@ARGV) {
  usage(0) if $ARGV[0] eq '--help';
  if ($ARGV[0] eq '--exit' || $ARGV[0] eq '--stop') {
    shift @ARGV;
    $exitrestart = 'exit';
    next;
  }
  if ($ARGV[0] eq '--restart') {
    shift @ARGV;
    $exitrestart = 'restart';
    next;
  }
  if ($ARGV[0] eq '--root') {
    shift @ARGV;
    $buildroot = shift @ARGV;
    next;
  }
  if ($ARGV[0] eq '--port') {
    shift @ARGV;
    $port = shift @ARGV;
    next;
  }
  if ($ARGV[0] eq '--arch') {
    shift @ARGV;
    $hostarch = shift @ARGV;
    next;
  }
  if ($ARGV[0] eq '--statedir') {
    shift @ARGV;
    $statedir = shift @ARGV;
    next;
  }
  if ($ARGV[0] eq '--srcserver') {
    # default value used if buildinfo does not contain srcserver element
    shift @ARGV;
    $srcserver = shift @ARGV;
    next;
  }
  if ($ARGV[0] eq '--reposerver') {
    shift @ARGV;
    my $server = shift @ARGV;
    push @reposervers, $server unless grep {$_ eq $server} @reposervers;
    next;
  }
  if ($ARGV[0] eq '--id') {
    shift @ARGV;
    $workerid = shift @ARGV;
    next;
  }
  if ($ARGV[0] eq '--test') {
    shift @ARGV;
    $testmode = 1;
    next;
  }
  if ($ARGV[0] eq '--kvm') {
    $vm = ' --kvm';
    shift @ARGV;
    next;
  }
  if ($ARGV[0] eq '--xen') {
    $vm = ' --xen';
    shift @ARGV;
    next;
  }
  if ($ARGV[0] eq '--lxc') {
    $vm = 'lxc';
    shift @ARGV;
    next;
  }
  if ($ARGV[0] eq '--tmpfs') {
    shift @ARGV;
    $vm_tmpfs_mode = 1;
    next;
  }
  if ($ARGV[0] eq '--xendevice' || $ARGV[0] eq '--device') {
    shift @ARGV;
    $vm_root = shift @ARGV;
    next;
  }
  if ($ARGV[0] eq '--xenswap' || $ARGV[0] eq '--swap') {
    shift @ARGV;
    $vm_swap = shift @ARGV;
    next;
  }
  if ($ARGV[0] eq '--vm-kernel') {
    shift @ARGV;
    $vm_kernel = shift @ARGV;
    next;
  }
  if ($ARGV[0] eq '--vm-initrd') {
    shift @ARGV;
    $vm_initrd = shift @ARGV;
    next;
  }
  if ($ARGV[0] eq '--vm-memory') {
    shift @ARGV;
    $vm_memory = shift @ARGV;
    next;
  }
  if ($ARGV[0] eq '--vmdisk-rootsize') {
    shift @ARGV;
    $vmdisk_rootsize = shift @ARGV;
    next;
  }
  if ($ARGV[0] eq '--vmdisk-swapsize') {
    shift @ARGV;
    $vmdisk_swapsize = shift @ARGV;
    next;
  }
  if ($ARGV[0] eq '--vmdisk-filesystem') {
    shift @ARGV;
    $vmdisk_filesystem = shift @ARGV;
    next;
  }
  if ($ARGV[0] eq '--vmdisk-mount-options') {
    shift @ARGV;
    $vmdisk_mount_options = shift @ARGV;
    next;
  }
  if ($ARGV[0] eq '--hugetlbfs') {
    shift @ARGV;
    $hugetlbfs = shift @ARGV;
    next;
  }
  if ($ARGV[0] eq '--build') {
    shift @ARGV;
    $justbuild = 1;
    next;
  }
  if ($ARGV[0] eq '--oneshot') {
    shift @ARGV;
    $oneshot = shift @ARGV;
    next;
  }
  if ($ARGV[0] eq '--hostcheck') {
    shift @ARGV;
    $hostcheck = shift @ARGV;
    next;
  }
  if ($ARGV[0] eq '--nocodeupdate') {
    shift @ARGV;
    $noworkercheck = 1;
    $nobuildcodecheck = 1;
    next;
  }
  if ($ARGV[0] eq '--noworkerupdate') {
    shift @ARGV;
    $noworkercheck = 1;
    next;
  }
  if ($ARGV[0] eq '--nobuildupdate') {
    shift @ARGV;
    $nobuildcodecheck = 1;
    next;
  }
  if ($ARGV[0] eq '--silent') {
    shift @ARGV;
    $silent= 1;
    next;
  }
  if ($ARGV[0] eq '--localkiwi') {
    shift @ARGV;
    $localkiwi = shift @ARGV;
    next;
  }
  if ($ARGV[0] eq '--jobs') {
    shift @ARGV;
    $jobs = shift @ARGV;
    next;
  }
  if ($ARGV[0] eq '--cachedir') {
    shift @ARGV;
    $cachedir = shift @ARGV;
    next;
  }
  if ($ARGV[0] eq '--cachesize') {
    shift @ARGV;
    $cachesize = shift @ARGV;
    $cachesize *= 1024*1024;
    next;
  }
  last;
}

if ($exitrestart) {
  usage(1) unless $statedir;
  if ((! -e "$statedir/lock") || BSUtil::lockcheck('>>', "$statedir/lock")) {
    die("worker not running.\n") if $exitrestart eq 'restart';
    print "worker not running.\n";
    exit(0);
  }
  my $state = lockstate();
  $state->{'state'} = 'idle' unless $state->{'state'};
  if ($state->{'state'} eq 'building' || $state->{'state'} eq 'killed' || $state->{'state'} eq 'discarded') {
    $state->{'state'} = 'discarded';
    $state->{'nextstate'} = $exitrestart eq 'exit' ? 'exit' : 'rebooting';
  } else {
    $state->{'state'} = $exitrestart eq 'exit' ? 'exit' : 'rebooting';
  }
  commitstate($state);
  # now wait till the state changes...
  while(1) {
    select(undef, undef, undef, .1);
    if (defined($exitrestart_timeout)) {
      $exitrestart_timeout -= .1;
      die("timeout reached, worker not responding\n") if $exitrestart_timeout < 0;
    }
    my $curstate = readxml("$statedir/state", $BSXML::workerstate, 1);
    last unless $curstate && $curstate->{'state'};
    next if $curstate->{'nextstate'};
    last if $curstate->{'state'} ne 'exit' && $curstate->{'state'} ne 'rebooting';
  }
  exit(0);
}

usage(1) unless $buildroot && $statedir;
usage(1) if ($cachedir && !$cachesize) || ($cachesize && !$cachedir);

$vm_root = "$buildroot/root" unless $vm_root;
$vm_swap = "$buildroot/swap" unless $vm_swap;

# here's the build code we want to use
$::ENV{'BUILD_DIR'} = "$statedir/build";

if (!$hostarch) {
  $hostarch = `uname -m`;
  chomp $hostarch;
  die("could not determine hostarch\n") unless $hostarch;
}

die("arch $hostarch cannot build anything!\n") unless $BSCando::cando{$hostarch} || ($hostarch eq 'local' && $localkiwi);

$srcserver = $BSConfig::srcserver unless defined $srcserver;
@reposervers = @BSConfig::reposervers unless @reposervers;

if ($justbuild) {
  my $buildinfo = readxml($ARGV[0], $BSXML::buildinfo);
  if ($localkiwi) {
    # make sure this is the right job for us
    die("not a kiwi job\n") unless $buildinfo->{'file'} =~ /\.kiwi$/;
    die("not a kiwi product job\n") unless $buildinfo->{'imagetype'} && $buildinfo->{'imagetype'}->[0] eq 'product';
  }
  $| = 1;

  dobuild($buildinfo);
  exit(0);
}

sub stream_logfile {
  my ($nostream, $start, $end) = @_;
  open(F, "<$buildroot/.build.log") || die("$buildroot/.build.log: $!\n");
  my @s = stat(F);
  $start ||= 0;
  if (defined($end)) {
    $end -= $start;
    die("end is smaller than start\n") if $end < 0;
  }
  die("Logfile is not that big\n") if $s[7] < abs($start);
  defined(sysseek(F, $start, $start < 0 ? Fcntl::SEEK_END : Fcntl::SEEK_SET)) || die("sysseek: $!\n");

  BSServer::reply(undef, 'Content-Type: text/plain', 'Transfer-Encoding: chunked');
  my $pos = sysseek(F, 0, Fcntl::SEEK_CUR) || die("sysseek: $!\n");
  while(!defined($end) || $end) {
    @s = stat(F);
    if ($s[7] <= $pos) {
      last if !$s[3];
      select(undef, undef, undef, .5);
      next;
    }
    my $data = '';
    my $l = $s[7] - $pos;
    $l = 4096 if $l > 4096;
    sysread(F, $data, $l);
    next unless length($data);
    $data = substr($data, 0, $end) if defined($end) && length($data) > $end;
    $pos += length($data);
    $end -= length($data) if defined $end;
    $data = sprintf("%X\r\n", length($data)).$data."\r\n";
    BSServer::swrite($data);
    last if $nostream && $pos >= $s[7];
  }
  close F;
  BSServer::swrite("0\r\n\r\n");
}

sub send_state {
  my ($state, $p, $ba, $exclude) = @_;
  my @args = ("state=$state", "arch=$ba", "port=$p");
  push @args, "workerid=$workerid" if defined $workerid;
  for my $server (@reposervers) {
    next if $exclude && $server eq $exclude;
    if ($state eq 'idle' && @reposervers > 1) {
      my $curstate = readxml("$statedir/state", $BSXML::workerstate, 1);
      last if $curstate && $curstate->{'state'} ne $state;
    }
    eval {
      BSRPC::rpc({
        'uri' => "$server/worker",
	'timeout' => 3,
      }, undef, @args);
    };
    print "send_state $server: $@" if $@;
  }
}

sub codemd5 {
  my ($dir) = @_;
  my @files = ls($dir);
  my $md5 = '';
  for my $file (sort @files) {
    next if -l "$dir/$file" || -d "$dir/$file";
    $md5 .= Digest::MD5::md5_hex(readstr("$dir/$file"))."  $file\n";
  }
  $md5 = Digest::MD5::md5_hex($md5);
  return $md5;
}

sub getcode {
  my ($dir, $uri, $ineval) = @_;

  # evalize ourself
  if (!$ineval) {
    my $md5;
    eval {
     $md5 = getcode($dir, $uri, 1);
    };
    if ($@) {
      warn($@);
      return '';
    }
    return $md5;
  }

  my $ndir = "$dir.new";
  my $odir = "$dir.old";

  # clean up stale runs
  if (-e $ndir) {
    unlink("$ndir/$_") for ls($ndir);
    rmdir($ndir) || die("rmdir $ndir: $!\n");
  }
  if (-e $odir) {
    unlink("$odir/$_") for ls($odir);
    rmdir($odir) || die("rmdir $odir: $!\n");
  }

  mkdir($ndir) || die("mkdir $ndir: $!\n");
  my $res = BSRPC::rpc({
    'uri' => $uri,
    'directory' => $ndir,
    'timeout' => $gettimeout,
    'withmd5' => 1,
    'receiver' => \&BSHTTP::cpio_receiver,
  });
  die("getcode error\n") unless $res;

  # got everything, clean things up, check if it really works
  if ($dir eq 'worker') {
    symlink('.', "$ndir/XML") || die("symlink: $!\n");
    chmod(0755, "$ndir/bs_worker");
    die("bs_worker selftest failed\n") if system("cd $ndir && ./bs_worker --selftest");
  } elsif ($dir eq 'build') {
    symlink('.', "$ndir/Build") || die("symlink: $!\n");
    symlink('.', "$ndir/Date") || die("symlink: $!\n");
    symlink('.', "$ndir/Time") || die("symlink: $!\n");
    # we just change everyfile to be on the safe side
    chmod(0755, "$ndir/$_->{'name'}") for @$res;
  }

  # ok, commit
  if (-e $dir) {
    rename($dir, $odir) || die("rename $dir $odir: $!\n");
  }
  rename($ndir, $dir) || die("rename $ndir $dir: $!\n");
  if (-e $odir) {
    unlink("$odir/$_") for ls($odir);
    rmdir($odir);
  }
  my $md5 = '';
  for my $file (sort {$a->{'name'} cmp $b->{'name'}} @$res) {
    $md5 .= "$file->{'md5'}  $file->{'name'}\n";
  }
  $md5 = Digest::MD5::md5_hex($md5);
  return $md5;
}


sub rm_rf {
  my ($dir) = @_;
  BSUtil::cleandir($dir);
  rmdir($dir);
  die("rmdir $dir failed\n") if -d $dir;
}

sub importbuild {
    return if defined &Build::queryhdrmd5;
    unshift @INC, "$statedir/build";
    require Build;
    Build->import();
}

sub getsources {
  my ($buildinfo, $dir) = @_;

  my @meta;
  push @meta, ($buildinfo->{'verifymd5'} || $buildinfo->{'srcmd5'})."  $buildinfo->{'package'}";
  my $server = $buildinfo->{'srcserver'} || $srcserver;

  my $res = BSRPC::rpc({
    'uri' => "$server/getsources",
    'directory' => $dir,
    'timeout' => $gettimeout,
    'withmd5' => 1,
    'receiver' => \&BSHTTP::cpio_receiver,
  }, undef, "project=$buildinfo->{'project'}", "package=$buildinfo->{'package'}", "srcmd5=$buildinfo->{'srcmd5'}");
  die("Error\n") unless ref($res) eq 'ARRAY';
  if (-e "$dir/.errors") {
    my $errors = readstr("$dir/.errors", 1);
    die("getsources: $errors");
  }
  # verify sources
  my %res = map {$_->{'name'} => $_} @$res;
  my $md5 = '';
  my @f = ls($dir);
  for my $f (sort @f) {
    die("unexpected file: $f") unless $res{$f};
    $md5 .= "$res{$f}->{'md5'}  $f\n";
  }
  $md5 = Digest::MD5::md5_hex($md5);
  die("source verification fails: $md5 != $buildinfo->{'verifymd5'}\n") if $md5 ne $buildinfo->{'verifymd5'};

  return @meta unless $buildinfo->{'file'} =~ /\.kiwi$/;

  # get additional kiwi sources
  my @sdep = grep {($_->{'repoarch'} || '') eq 'src'} @{$buildinfo->{'bdep'} || []};
  for my $src (@sdep) {
    print "$src->{'name'}, ";
    my $idir = "$src->{'project'}/$src->{'package'}";
    $idir = "$dir/images/$idir";
    mkdir_p($idir);
    my $res = BSRPC::rpc({
      'uri' => "$server/getsources",
      'directory' => $idir,
      'timeout' => $gettimeout,
      'withmd5' => 1,
      'receiver' => \&BSHTTP::cpio_receiver,
    }, undef, "project=$src->{'project'}", "package=$src->{'package'}", "srcmd5=$src->{'srcmd5'}");
    die("Error\n") unless ref($res) eq 'ARRAY';
    if (-e "$idir/.errors") {
      my $errors = readstr("$idir/.errors", 1);
      die("getsources: $errors");
    }
    push @meta, "$src->{'srcmd5'}  $src->{'project'}/$src->{'package'}";
  }
  return @meta;
}

sub getdeltasources {
  my ($buildinfo, $dir) = @_;

  my @meta;
  push @meta, ($buildinfo->{'verifymd5'} || $buildinfo->{'srcmd5'})."  $buildinfo->{'package'}";
  my $server = $buildinfo->{'reposerver'};
  my $res = BSRPC::rpc({
    'uri' => "$server/getjobdata",
    'directory' => $dir,
    'timeout' => $gettimeout,
    'receiver' => \&BSHTTP::cpio_receiver,
  }, undef, "job=$buildinfo->{'job'}", "arch=$buildinfo->{'arch'}", "jobid=$buildinfo->{'jobid'}");
  die("Error\n") unless ref($res) eq 'ARRAY';
  return @meta;
}

sub qsystem {
  my (@args) = @_;

  my $pid;
  if (!($pid = xfork())) {
    $SIG{'PIPE'} = 'DEFAULT';
    open(STDOUT, ">/dev/null") if $silent;
    exec(@args);
    die("$args[0]: $!\n"); 
  }
  waitpid($pid, 0) == $pid || die("waitpid $pid: $!\n"); 
  return $?;
}

sub link_or_copy {
  my ($from, $to) = @_;
  return 1 if link($from, $to);
  local *F;
  local *G;
  return undef unless open(F, '<', $from);
  if (!open(G, '>', $to)) {
    close F;
    return undef;
  }
  my $buf;
  while (sysread(F, $buf, 8192)) {
    (syswrite(G, $buf) || 0) == length($buf) || die("$to write: $!\n");
  }
  close(F);
  if (!close(G)) {
    unlink($to);
    return undef;
  }
  return 1;
}

sub manage_cache {
  my ($prunesize, $cacheold, $cachenew) = @_;
  # get the lock
  local *F;
  BSUtil::lockopen(\*F, '+>>', "$cachedir/content", 1) || return;
  my $content;
  if (-s F) {
    seek(F, 0, 0);
    $content = Storable::fd_retrieve(\*F);
  }
  $content ||= [];
  my %content = map {$_->[0] => $_->[1]} @$content;
  # put cacheold, cachenew at the top
  if ($cacheold && @$cacheold) {
    splice(@$content, 0, 0, @$cacheold);
    $content{$_->[0]} = $_->[1] for @$cacheold;
  }
  if ($cachenew) {
    for my $c (reverse @$cachenew) {
      my $path = pop(@$c);
      my $cacheid = $c->[0];
      my $cachefile = "$cachedir/".substr($cacheid, 0, 2)."/$cacheid";
      mkdir_p("$cachedir/".substr($cacheid, 0, 2));
      unlink("$cachefile.$$");
      next unless link_or_copy($path, "$cachefile.$$");
      rename("$cachefile.$$", $cachefile) || die("rename $cachefile.$$ $cachefile: $!\n");
      my $mpath = "$path.meta";
      $mpath = "$1.meta" if $path =~ /^(.*)\.(?:$binsufsre)$/;
      if (-s $mpath) {
	unlink("$cachefile.meta.$$");
	if (link_or_copy($mpath, "$cachefile.meta.$$")) {
	  rename("$cachefile.meta.$$", "$cachefile.meta") || die("rename $cachefile.meta.$$ $cachefile.meta: $!\n");
	} else {
	  unlink("$cachefile.meta");
	}
      } else {
	unlink("$cachefile.meta");
      }
      unshift @$content, $c;
      $content{$c->[0]} = $c->[1];
    }
  }
  # prune cache
  for my $c (@$content) {
    if (!defined delete $content{$c->[0]}) {
      $c = undef;
      next;
    }
    $prunesize -= $c->[1];
    if ($prunesize < 0) {
      my $cacheid = $c->[0];
      my $cachefile = "$cachedir/".substr($cacheid, 0, 2)."/$cacheid";
      unlink($cachefile);
      unlink("$cachefile.meta");
      $c = undef;
      next;
    }
  }
  @$content = grep {defined $_} @$content;
  Storable::nstore($content, "$cachedir/content.new");
  rename("$cachedir/content.new", "$cachedir/content") || die("rename $cachedir/content.new $cachedir/content");
  close F;
}

sub getbinaries_cache {
  my ($dir, $server, $projid, $repoid, $arch, $nometa, $bins, $bvl) = @_;

  importbuild() unless defined &Build::queryhdrmd5;
  if (! -d $dir) {
    mkdir_p($dir) || die("mkdir_p $dir: $!\n");
  }
  my %ret;
  undef $bvl unless $cachedir;	# no use here if ther's no cachedir
  if ($bvl) {
    # check if it contains everything we want to know
    my %bv;
    for (@{$bvl->{'binary'} || []}) {
      if ($_->{'error'}) {
        $bv{$_->{'name'}} = 1;
      } else {
        $bv{$1} = 1 if $_->{'name'} =~ /(.*)\.(?:$binsufsre)$/;
      }
    }
    undef $bvl if grep {!$bv{$_}} @$bins;
  }
  if ($cachedir && !$bvl) {
    my @args;
    push @args, "project=$projid";
    push @args, "repository=$repoid";
    push @args, "arch=$arch";
    push @args, "nometa" if $nometa;
    push @args, "binaries=".join(',', @$bins);
    eval {
      $bvl = BSRPC::rpc({
        'uri' => "$server/getbinaryversions",
        'timeout' => $gettimeout,
        }, $BSXML::binaryversionlist, @args);
    };
    warn($@) if $@;
  }
  $bvl ||= {};
  my %bv;
  for (@{$bvl->{'binary'} || []}) {
    if ($_->{'error'}) {
      $bv{$_->{'name'}} = $_;
    } else {
      $bv{$1} = $_ if $_->{'name'} =~ /(.*)\.(?:$binsufsre)$/;
    }
  }
  my @downloadbins;
  my $downloadsizek = 0;
  my @cacheold;
  my @cachenew;
  for my $bin (@$bins) {
    my $bv = $bv{$bin};
    if (!$bv) {
      push @downloadbins, $bin;
      next;
    }
    next if $bv->{'error'};
    my $cacheid =  Digest::MD5::md5_hex("$projid/$repoid/$arch/$bv->{'hdrmd5'}");
    my $cachefile = "$cachedir/".substr($cacheid, 0, 2)."/$cacheid";
    my $usecache = 0;
    my $havemeta;
    if (link_or_copy($cachefile, "$dir/$bv->{'name'}")) {
      my @s = stat("$dir/$bv->{'name'}");
      die unless @s;
      if (!$nometa) {
	my $mn = $bv->{'name'};
        $mn =~ s/\.(?:$binsufsre)/.meta/;
        if (link_or_copy("$cachefile.meta", "$dir/$mn")) {
	  local *F;
	  open(F, '<', "$dir/$mn") || die;
	  my $ctx = Digest::MD5->new;
	  $ctx->addfile(*F);
	  close F;
	  if ($ctx->hexdigest() eq $bv->{'metamd5'}) {
	    $usecache = 1;
	    $havemeta = 1;
	  } else {
	    unlink("$dir/$mn");
	  }
	}
      } else {
	$usecache = 1;
      }
      if ($usecache) {
	# check hdrmd5 to be sure we got the right bin
        my $id = Build::queryhdrmd5("$dir/$bv->{'name'}");
	$usecache = 0 if ($id || '') ne $bv->{'hdrmd5'};
      }
      if (!$usecache) {
	unlink("$dir/$bv->{'name'}");
      } else {
	push @cacheold, [$cacheid, $s[7]];
      }
    }
    if (!$usecache) {
      push @downloadbins, $bin;
      $downloadsizek += $bv->{'sizek'};
    } else {
      $ret{$bin} = {'name' => $bv->{'name'}, 'hdrmd5' => $bv->{'hdrmd5'}};
      if (!$nometa && $havemeta) {
        $ret{$bin}->{'meta'} = 1;
      }
    }
  }
  #print "(cache: ".@cacheold." hits, ".@downloadbins." misses)";
  if (@downloadbins) {
    if ($cachedir && $downloadsizek * 1024 * 100 > $cachesize) {
      # reserve space
      manage_cache($cachesize - $downloadsizek * 1024);
    }
    my @args;
    push @args, "project=$projid";
    push @args, "repository=$repoid";
    push @args, "arch=$arch";
    push @args, "binaries=".join(',', @downloadbins);
    my $res = BSRPC::rpc({
      'uri' => "$server/getbinaries",
      'directory' => $dir,
      'timeout' => $gettimeout,
      'receiver' => \&BSHTTP::cpio_receiver,
    }, undef, @args);
    die("Error\n") unless ref($res) eq 'ARRAY';
    my %havemeta;
    for my $r (@$res) {
      if ($r->{'name'} =~ /^(.*)\.(?:$binsufsre)$/) {
	my $n = $1;
	my @s = stat("$dir/$r->{'name'}");
	die unless @s;
        my $id = Build::queryhdrmd5("$dir/$r->{'name'}");
	$r->{'hdrmd5'} = $id;
	my $cacheid =  Digest::MD5::md5_hex("$projid/$repoid/$arch/$id");
	push @cachenew, [$cacheid, $s[7], "$dir/$r->{'name'}"];
	$ret{$n} = $r;
      } elsif ($r->{'name'} =~ /^(.*)\.meta$/) {
        $havemeta{$1} = 1;
      }
    }
    for (keys %havemeta) {
      next unless $ret{$_};
      $ret{$_}->{'meta'} = 1;
    }
  }
  manage_cache($cachesize, \@cacheold, \@cachenew) if $cachedir;
  if ($nometa) {
    for (keys %ret) {
      next unless $ret{$_}->{'meta'};
      unlink("$dir/$_.meta");
      delete $ret{$_}->{'meta'};
    }
  }
  return \%ret;
}

sub getbinaries_kiwiproduct {
  my ($buildinfo, $dir, $srcdir) = @_;

  # we need the Build package for queryhdrmd5
  importbuild() unless defined &Build::queryhdrmd5;

  # create list of prpaps
  my @kdeps;
  my %prpaps;
  my %linkit;
  my %prpapackages;
  my %packagebinaryversionlist;
  for my $dep (@{$buildinfo->{'bdep'} || []}) {
    if (!defined($dep->{'package'})) {
      push @kdeps, $dep->{'name'};
      next;
    }
    my $repoarch = $dep->{'repoarch'} || $buildinfo->{'arch'};
    next if $repoarch eq 'src';
    if (!$prpaps{"$dep->{'project'}/$dep->{'repository'}/$repoarch/$dep->{'package'}"}) {
      push @{$prpapackages{"$dep->{'project'}/$dep->{'repository'}/$repoarch"}}, $dep->{'package'};
      $prpaps{"$dep->{'project'}/$dep->{'repository'}/$repoarch/$dep->{'package'}"} = 1;
    }
    $linkit{"$dep->{'project'}/$dep->{'repository'}/$repoarch/$dep->{'package'}/$dep->{'name'}"} = 1 unless $dep->{'noinstall'};
  }

  my %prp2server;
  for (@{$buildinfo->{'path'} || []}) {
    $prp2server{"$_->{'project'}/$_->{'repository'}"} = $_->{'server'};
  }

  mkdir_p($dir);

  # fetch packages needed for product building
  for my $repo (@{$buildinfo->{'syspath'} || $buildinfo->{'path'} || []}) {
    last if !@kdeps;
    my $repoarch = $buildinfo->{'arch'};
    $repoarch = $BSConfig::localarch if $repoarch eq 'local' && $BSConfig::localarch;
    my $server = $repo->{'server'} || $buildinfo->{'reposerver'};
    my $got = getbinaries_cache($dir, $server, $repo->{'project'}, $repo->{'repository'}, $repoarch, 1, \@kdeps);
    @kdeps = grep {!$got->{$_}} @kdeps;
  }
  die("getbinaries_kiwiproduct: missing packages: @kdeps\n") if @kdeps;

  my %meta;
  my $linklocal = $localkiwi && -d "$localkiwi/build" ? 1 : 0;

  my %cachenew;
  my @cacheold;
  my $downloadsizek = 0;

  for my $prpap (sort keys %prpaps) {
    my ($projid, $repoid, $arch, $packid) = split('/', $prpap, 4);
    my $prpdir = "$projid/$repoid";
    my $ddir = "$srcdir/repos/$prpdir";
    mkdir_p($ddir);
    my $res;
    my $server =  $prp2server{"$projid/$repoid"} || $buildinfo->{'reposerver'};
    my %knownmd5;
    if ($linklocal) {
      $res = [];
      for my $name (ls("$localkiwi/build/$projid/$repoid/$arch/$packid")) {
	my $rarch;
	if ($name =~ /-[^-]+-[^-]+\.([a-zA-Z][^\.\-]*)\.rpm$/) {
	  $rarch = $1;
	} elsif ($name =~ /appdata.xml$/) {
	  $rarch = $arch; 
	}
	next unless $rarch;
	mkdir_p("$ddir/$rarch") unless -d "$ddir/$rarch";
	if (!link("$localkiwi/build/$projid/$repoid/$arch/$packid/$name", "$ddir/$rarch/$name")) {
	  # link fails if file already exists, rpc just overwrites
	  unlink("$ddir/$rarch/$name");
	  link("$localkiwi/build/$projid/$repoid/$arch/$packid/$name", "$ddir/$rarch/$name") || die("link $localkiwi/build/$projid/$repoid/$arch/$packid/$name $ddir/$rarch/$name: $!\n");
	}
	push @$res, {'name' => "$rarch/$name"};
      }
    } else {
      my $bvl;
      if ($cachedir) {
	if ($server eq $buildinfo->{'reposerver'}) {
	  my $pbvl = $packagebinaryversionlist{"$projid/$repoid/$arch"};
	  if (!$pbvl) {
	    eval {
	      $pbvl = BSRPC::rpc({
		'uri' => "$server/getpackagebinaryversionlist",
		'timeout' => $gettimeout,
	      }, $BSXML::packagebinaryversionlist, "project=$projid", "repository=$repoid", "arch=$arch", map {"package=$_"} @{$prpapackages{"$projid/$repoid/$arch"}});
	    };
	    undef $pbvl if $@;
	    warn($@) if $@;
	    $pbvl = { map {$_->{'package'} => $_} @{$pbvl->{'binaryversionlist'} || []} } if $pbvl;
	    $pbvl ||= {};
	    $packagebinaryversionlist{"$projid/$repoid/$arch"} = $pbvl;
	  }
          $bvl = $pbvl->{$packid};
	}
	if (!defined($bvl)) {
	  eval {
	    $bvl = BSRPC::rpc({
	      'uri' => "$server/build/$projid/$repoid/$arch/$packid",
	      'timeout' => $gettimeout,
	    }, $BSXML::binaryversionlist, 'view=binaryversions');
	  };
	  undef $bvl if $@;
	  warn($@) if $@;
	}
      }
      my @good;
      my @bad;
      if ($bvl) {
	for my $bv (@{$bvl->{'binary'} || []}) {
	  my $bin = $bv->{'name'};
	  next unless $bin =~ /-[^-]+-[^-]+\.([a-zA-Z][^\.\-]*)\.rpm$/;
	  my $rarch = $1;
	  my $cacheid =  Digest::MD5::md5_hex("$projid/$repoid/$arch/$bv->{'hdrmd5'}");
	  my $cachefile = "$cachedir/".substr($cacheid, 0, 2)."/$cacheid";
	  mkdir_p("$ddir/$rarch");
	  if (link_or_copy($cachefile, "$ddir/$rarch/$bin.new.rpm")) {
	    my @s = stat("$ddir/$rarch/$bin.new.rpm");
	    die unless @s;
	    my $leadsigmd5 = '';
	    my $id = Build::queryhdrmd5("$ddir/$rarch/$bin.new.rpm", \$leadsigmd5);
	    if ($id eq $bv->{'hdrmd5'} && (!$bv->{'leadsigmd5'} || $bv->{'leadsigmd5'} eq $leadsigmd5)) {
	      push @good, "$rarch/$bin";
	      push @cacheold, [$cacheid, $s[7]];
	      $knownmd5{"$rarch/$bin"} = $id;
	    } else {
	      unlink "$ddir/$rarch/$bin.new.rpm";
	      push @bad, $bv;
	      $downloadsizek += $bv->{'sizek'};
	    }
	  } else {
	    push @bad, $bv;
	    $downloadsizek += $bv->{'sizek'};
	  }
	}
      }
      #print "(cache: ".@good." hits, ".@bad." misses)";
      if ($bvl && @bad) {
	if ($cachedir && $downloadsizek * 1024 * 100 > $cachesize) {
	  # reserve space
	  manage_cache($cachesize - $downloadsizek * 1024, \@cacheold, [ values %cachenew ]);
	  @cacheold = ();
	  %cachenew = ();
	  $downloadsizek = 0;
	}
	eval {
	  BSRPC::rpc({
	    'uri' => "$server/build/$projid/$repoid/$arch/$packid",
	    'directory' => $ddir,
	    'timeout' => $gettimeout,
	    'map' => sub {
	      my ($param, $name) = @_;
	      if ($name =~ /-[^-]+-[^-]+\.([a-zA-Z][^\.\-]*)\.rpm$/) {
		my $rarch = $1;
		mkdir_p("$param->{'directory'}/$rarch");
		$name = "$rarch/$name.new.rpm";
	      }
	      return $name;
	    },
	    'receiver' => \&BSHTTP::cpio_receiver,
	  }, undef, 'view=cpio', map {"binary=$_->{'name'}"} @bad);
	};
	if ($@) {
	  # delete everything as a file might be incomplete
          for my $bv (@bad) {
	    my $bin = $bv->{'name'};
	    next unless $bin =~ /-[^-]+-[^-]+\.([a-zA-Z][^\.\-]*)\.rpm$/;
            my $rarch = $1;
	    unlink("$ddir/$rarch/$bin.new.rpm");
	  }
	}
        for my $bv (splice @bad) {
	  my $bin = $bv->{'name'};
	  next unless $bin =~ /-[^-]+-[^-]+\.([a-zA-Z][^\.\-]*)\.rpm$/;
          my $rarch = $1;
	  my @s = stat("$ddir/$rarch/$bin.new.rpm");
	  if (!@s) {
	    push @bad, $bv;
	    next;
	  }
	  my $leadsigmd5 = '';
	  my $id = Build::queryhdrmd5("$ddir/$rarch/$bin.new.rpm", \$leadsigmd5);
	  if ($id eq $bv->{'hdrmd5'} && (!$bv->{'leadsigmd5'} || $bv->{'leadsigmd5'} eq $leadsigmd5)) {
	    push @good, "$rarch/$bin";
            my $cacheid =  Digest::MD5::md5_hex("$projid/$repoid/$arch/$bv->{'hdrmd5'}");
	    $cachenew{"$ddir/$rarch/$bin"} = [$cacheid, $s[7], "$ddir/$rarch/$bin"];
	    $knownmd5{"$rarch/$bin"} = $id;
	  } else {
	    unlink "$ddir/$rarch/$bin.new.rpm";
	    push @bad, $bv;
	  }
	}
        #print "(still ".@bad." misses)";
      }
      if ($bvl && !@bad) {
	for (@good) {
	  rename("$ddir/$_.new.rpm", "$ddir/$_") || die("rename $ddir/$_.new.rpm $ddir/$_: $!\n");
	}
        $res = [ map {{'name' => $_}} @good ];
      } else {
	%knownmd5 = ();
        for (@good) {
	  unlink("$ddir/$_.new.rpm");
	  delete $cachenew{"$ddir/$_"};
	}
	$res = BSRPC::rpc({
	  'uri' => "$server/build/$projid/$repoid/$arch/$packid",
	  'directory' => $ddir,
	  'timeout' => $gettimeout,
	  'map' => sub {
	    my ($param, $name) = @_;
	    if ($name =~ /-[^-]+-[^-]+\.([a-zA-Z][^\.\-]*)\.rpm$/) {
	      my $rarch = $1;
	      mkdir_p("$param->{'directory'}/$rarch");
	      $name = "$rarch/$name";
	    } else { 
	      $name = "$arch/$name";
	    }
	    return $name;
	  },
	  'receiver' => \&BSHTTP::cpio_receiver,
	}, undef, 'view=cpio');
	if ($cachedir) {
	  for my $f (@{$res || []}) {
	    my @s = stat("$ddir/$f->{'name'}");
	    next unless @s;
	    my $id = Build::queryhdrmd5("$ddir/$f->{'name'}");
	    next unless $id;
	    my $cacheid =  Digest::MD5::md5_hex("$projid/$repoid/$arch/$id");
	    $cachenew{"$ddir/$f->{'name'}"} = [$cacheid, $s[7], "$ddir/$f->{'name'}"];
	    $knownmd5{$f->{'name'}} = $id;
	  }
	  #print "(put ".@cachenew." entries)";
	}
      }
    }
    die unless $res;
    my @todometa;
    my @todometa_f;
    for my $f (@$res) {
      my (undef, $name) = split('/', $f->{'name'}, 2);
      next unless defined($name) && $name =~ /^(.*)-[^-]+-[^-]+\.([a-zA-Z][^\.\-]*)\.rpm$/;
      my ($n, $rarch) = ($1, $2);
      my $id = $knownmd5{$f->{'name'}};
      $id ||= Build::queryhdrmd5("$ddir/$f->{'name'}");
      $id ||= 'deaddeaddeaddeaddeaddeaddeaddead';
      if ($linkit{"$prpap/$n"} && $rarch ne 'src' && $rarch ne 'nosrc') {
        link("$ddir/$f->{'name'}", "$dir/$n.rpm") unless -e "$dir/$n.rpm";
      }
      $meta{"$prpap/$n.$rarch"} = $id;
    }
  }

  manage_cache($cachesize, \@cacheold, [ values %cachenew ]) if $cachedir;

  # create meta
  my @meta;
  for (sort keys %meta) {
    push @meta, "$meta{$_}  $_";
  } 
  return @meta;
}

sub getpreinstallimage_metas {
  my ($buildinfo, $dir, $metas, $img, $hdrmd5s) = @_;

  my $prpa = "$buildinfo->{'project'}/$buildinfo->{'repository'}/$buildinfo->{'arch'}";

  # find out which metas are needed for this image
  my @bins;
  my %havehdrmd5 = map {$_ => 1} @{$img->{'hdrmd5s'} || []};
  for my $bin (sort(keys %$metas)) {
    push @bins, $bin if $hdrmd5s->{$bin} && $havehdrmd5{$hdrmd5s->{$bin}};
  }

  my @todo;
  if ($cachedir) {
    # check the cache
    for my $bin (@bins) {
      my $bv = $metas->{$bin};
      my $cacheid =  Digest::MD5::md5_hex("$prpa/$bv->{'hdrmd5'}");
      my $cachefile = "$cachedir/".substr($cacheid, 0, 2)."/$cacheid";
      if (link_or_copy("$cachefile.meta", "$dir/$bin.meta")) {
	local *F;
	open(F, '<', "$dir/$bin.meta") || die;
	my $ctx = Digest::MD5->new;
	$ctx->addfile(*F);
	close F;
	next if $ctx->hexdigest() eq $bv->{'metamd5'};
	unlink("$dir/$bin.meta");
      }
      push @todo, $bin;
    }
  } else {
    @todo = @bins;
  }
  return 1 unless @todo;

  # get missing ones for the repo server for this build
  my @args;
  push @args, "project=$buildinfo->{'project'}";
  push @args, "repository=$buildinfo->{'repository'}";
  push @args, "arch=$buildinfo->{'arch'}";
  push @args, "binaries=".join(',', @todo);
  push @args, "metaonly=1";
  my $repo = (grep {$_->{'project'} eq $buildinfo->{'project'} && $_->{'repository'} eq $buildinfo->{'repository'}} @{$buildinfo->{'path'} || []})[0] || {};
  my $server = $repo->{'server'} || $buildinfo->{'reposerver'};
  my $res;
  eval {
    $res = BSRPC::rpc({
      'uri' => "$server/getbinaries",
      'directory' => $dir,
      'timeout' => $gettimeout,
      'receiver' => \&BSHTTP::cpio_receiver,
    }, undef, @args);
    die("Error\n") unless ref($res);
  };
  warn($@) if $@;
  return undef unless $res;
  my %havemeta;
  for my $r (@$res) {
    next unless $r->{'name'} =~ /^(.*)\.meta$/;
    my $bin = $1;
    my $bv = $metas->{$bin};
    die("downloaded the wrong meta\n") unless $bv;
    local *F;
    next unless open(F, '<', "$dir/$bin.meta");
    my $ctx = Digest::MD5->new;
    $ctx->addfile(*F);
    close F;
    if ($ctx->hexdigest() eq $bv->{'metamd5'}) {
      # matching meta!
      $havemeta{$bin} = 1;
      if ($cachedir) {
	# put in cache so that we don't need to download it the next time
	my $cacheid =  Digest::MD5::md5_hex("$prpa/$bv->{'hdrmd5'}");
	my $cachefile = "$cachedir/".substr($cacheid, 0, 2)."/$cacheid";
	unlink("$cachefile.meta.$$");
	if (link_or_copy("$dir/$bin.meta", "$cachefile.meta.$$")) {
	  rename("$cachefile.meta.$$", "$cachefile.meta") || die("rename $cachefile.meta.$$ $cachefile.meta: $!\n");
	}
      }
    } else {
      unlink("$dir/$bin.meta");
    }
  }
  return (grep {!$havemeta{$_}} @todo) ? undef : 1;
}

sub getpreinstallimage {
  my ($buildinfo, $dir, $todo, $bvls) = @_;

  # get all bvls to calculate image data
  my @bins = @$todo;
  my %prpas;	# server -> prpas on this server
  my %hdrmd5s;
  my $projid = $buildinfo->{'project'};
  my $repoid = $buildinfo->{'repository'};
  my %metas;
  for my $repo (@{$buildinfo->{'path'} || []}) {
    my $server = $repo->{'server'} || $buildinfo->{'reposerver'};
    my $nometa = $repo->{'project'} ne $projid || $repo->{'repository'} ne $repoid ? 1 : 0;
    $nometa = 1 if $buildinfo->{'file'} eq '_preinstallimage';
    my @args;
    push @args, "project=$repo->{'project'}";
    push @args, "repository=$repo->{'repository'}";
    push @args, "arch=$buildinfo->{'arch'}";
    push @args, "nometa" if $nometa;
    push @args, "binaries=".join(',', @bins);
    my $bvl;
    eval {
      $bvl = BSRPC::rpc({
	'uri' => "$server/getbinaryversions",
	'timeout' => $gettimeout,
      }, $BSXML::binaryversionlist, @args);
    };
    warn($@) if $@;
    last unless $bvl;	# stop here
    push @{$prpas{$server}}, "$repo->{'project'}/$repo->{'repository'}/$buildinfo->{'arch'}" if $server ne $buildinfo->{'srcserver'};
    $bvls->{"$repo->{'project'}/$repo->{'repository'}/$buildinfo->{'arch'}"} = $bvl if $bvls;
    my %bv;
    for (@{$bvl->{'binary'} || []}) {
      if ($_->{'error'}) {
	$bv{$_->{'name'}} = $_;
      } else {
	next unless $_->{'name'} =~ /(.*)\.(?:$binsufsre)$/;
	$bv{$1} = $_;
	$metas{$1} = $_ if !$nometa && $_->{'metamd5'};
      }
    }
    for my $bin (@bins) {
      my $bv = $bv{$bin} || {};
      $hdrmd5s{$bin} = $bv->{'hdrmd5'} unless $bv->{'error'};
    }
    @bins = grep {!exists $hdrmd5s{$_}} @bins;
    last unless @bins;
  }
  # wipe error packages from hdrmd5s
  for (keys %hdrmd5s) {
    delete $hdrmd5s{$_} unless defined $hdrmd5s{$_};
  }
  return undef unless %hdrmd5s && %prpas;

  # ok, now check if there is an image on one of the repo servers
  my $match = "\0" x 512;
  for (values %hdrmd5s) {
    vec($match, hex(substr($_, 0, 3)), 1) = 1;
  }
  #$match = unpack("H*", $match);	# hexify
  my @images;
  for my $server (sort(keys %prpas)) {
    my $serverimages;
    eval { 
      $serverimages = BSRPC::rpc({
	'uri' => "$buildinfo->{'reposerver'}/getpreinstallimageinfos",
	'request' => 'POST',
	'data' => $match,
	'headers' => [ 'Content-Type: application/octet-stream' ],
	'timeout' => $gettimeout,
      }, undef, "match=body", map {"prpa=$_"} @{$prpas{$server}});
    };
    warn($@) if $@;
    if ($serverimages && substr($serverimages, 0, 4) eq 'pst0') {
      eval {
	$serverimages = Storable::thaw(substr($serverimages, 4));
      };
      warn($@) if $@;
    }
    $serverimages = [] unless $serverimages && ref($serverimages);
    $_->{'server'} = $server for @$serverimages;
    push @images, @$serverimages;
  }

  # got all available images, now find a good one to use
  my $imagefile;
  my $imageinfo;
  my %neededhdrmd5s;
  $neededhdrmd5s{$_} = 1 for values %hdrmd5s;
  while (@images) {
    my $bestimgn = 2;
    my $bestimg;
    for my $img (@images) {
      # check if it really fits!
      next if @{$img->{'hdrmd5s'} || []} < $bestimgn;
      next unless $img->{'sizek'} && $img->{'hdrmd5'};
      next if grep {!$neededhdrmd5s{$_}} @{$img->{'hdrmd5s'} || []};
      # ignore our own image
      next if $img->{'prpa'} eq "$projid/$repoid/$buildinfo->{'arch'}" && $img->{'package'} eq $buildinfo->{'package'};
      $bestimg = $img;
      $bestimgn = @{$img->{'hdrmd5s'} || []};
    }
    last unless $bestimg;	# nothing fits
    # try this one
    my $ifile = "preinstallimage.$bestimg->{'file'}";
    unlink("$dir/$ifile");

    # check if the image is in the cache
    my $cacheid;
    my $cachefile;
    if ($cachedir) {
      $cacheid =  Digest::MD5::md5_hex("$bestimg->{'prpa'}/$bestimg->{'hdrmd5'}");
      $cachefile = "$cachedir/".substr($cacheid, 0, 2)."/$cacheid";
      my $cachefilemeta = readstr("$cachefile.meta", 1) || '';
      if ($cachefilemeta eq "$bestimg->{'hdrmd5'}  :preinstallimage\n") {
	if (link_or_copy($cachefile, "$dir/$ifile")) {
	  # re-check to make races unlikely
	  $cachefilemeta = readstr("$cachefile.meta", 1) || '';
	  if ($cachefilemeta eq "$bestimg->{'hdrmd5'}  :preinstallimage\n") {
	    # looks good, use it!
	    my @s = stat("$dir/$ifile");
	    if (@s) {
	      # put entry on top
	      manage_cache($cachesize, [ [$cacheid, $s[7]] ], undef);
	    }
	    if (getpreinstallimage_metas($buildinfo, $dir, \%metas, $bestimg, \%hdrmd5s)) {
	      $imagefile = $ifile;
	      $imageinfo = $bestimg;
	      last;
	    }
	  }
	}
	unlink("$dir/$ifile");
      }
    }

    # nope, download it
    manage_cache($cachesize - $bestimg->{'sizek'} * 1024) if $cachedir;	# make room
    my $res;
    eval {
      $res = BSRPC::rpc({
	'uri' => "$bestimg->{'server'}/build/$bestimg->{'prpa'}/$bestimg->{'path'}",
	'timeout' => $gettimeout,
	'receiver' => \&BSHTTP::file_receiver,
	'filename' => "$dir/$ifile",
      });
    };
    if ($@) {
      warn($@);
    } elsif (-s "$dir/$ifile") {
      my @s = stat(_);
      if ($cachedir && @s) {
	# put in cache (and fake meta)
	writestr("$dir/$ifile.meta", undef, "$bestimg->{'hdrmd5'}  :preinstallimage\n");
	manage_cache($cachesize, undef, [ [$cacheid, $s[7], "$dir/$ifile"] ]);
	unlink("$dir/$ifile.meta");
      }
      if (getpreinstallimage_metas($buildinfo, $dir, \%metas, $bestimg, \%hdrmd5s)) {
        $imagefile = $ifile;
        $imageinfo = $bestimg;
        last;
      }
    }
    unlink("$dir/$ifile");

    # that did not turn out well, try another image
    $bestimg->{'hdrmd5s'} = [];	# disables this image
  }

  return undef unless $imagefile;

  my %imagebins;
  my %havehdrmd5 = map {$_ => 1} @{$imageinfo->{'hdrmd5s'} || []};
  for my $bin (keys %hdrmd5s) {
    $imagebins{$bin} = $hdrmd5s{$bin} if $havehdrmd5{$hdrmd5s{$bin}};
  }
  for my $bin (keys %metas) {
    delete $metas{$_} unless $imagebins{$bin};	# only return metas on the image
  }
  my $imagesource = $imageinfo->{'prpa'};
  $imagesource =~ s/\/[^\/]*$//;	# strip arch
  $imagesource .= "/$imageinfo->{'package'}" if $imageinfo->{'package'};
  $imagesource .= " [$imageinfo->{'hdrmd5'}]";
  return $imagefile, \%imagebins, $imagesource, \%metas;
}

sub getbinaries {
  my ($buildinfo, $dir, $srcdir, $preinstallimagedata) = @_;

  mkdir_p($dir);
  my $kiwimode;
  $kiwimode = 1 if $buildinfo->{'file'} =~ /\.kiwi$/;

  if ($kiwimode && $buildinfo->{'imagetype'} && $buildinfo->{'imagetype'}->[0] eq 'product') {
    return getbinaries_kiwiproduct($buildinfo, $dir, $srcdir);
  }

  # we need the Build package for queryhdrmd5
  importbuild() unless defined &Build::queryhdrmd5;

  my @bdep = @{$buildinfo->{'bdep'} || []};
  my %bdep_noinstall = map {$_->{'name'} => 1} grep {$_->{'noinstall'} && ($_->{'repoarch'} || '') ne 'src'} @bdep;
  my %bdep_notmeta = map {$_->{'name'} => 1} grep {$_->{'notmeta'} && ($_->{'repoarch'} || '') ne 'src'} @bdep;
  @bdep = map {$_->{'name'}} grep {($_->{'repoarch'} || '') ne 'src'} @bdep;

  my %done;
  my @todo = @bdep;
  die("no binaries needed for this package?\n") unless @todo;
  my @meta;
  my %meta;
  my $projid = $buildinfo->{'project'};
  my $repoid = $buildinfo->{'repository'};

  if ($kiwimode && $buildinfo->{'syspath'}) {
    # two path mode: fetch environment
    @todo = grep {!$bdep_noinstall{$_}} @bdep;
    for my $repo (@{$buildinfo->{'syspath'} || []}) {
      last if !@todo;
      my $server = $repo->{'server'} || $buildinfo->{'reposerver'};
      my $got = getbinaries_cache($dir, $server, $repo->{'project'}, $repo->{'repository'}, $buildinfo->{'arch'}, 1, \@todo);
      @todo = grep {!$got->{$_}} @todo;
    }
    die("getbinaries: missing packages: @todo\n") if @todo;
    @todo = grep {$bdep_noinstall{$_} || !$bdep_notmeta{$_}} @bdep;
  }

  my %bvls;		# bvl cache, the query is not free
  my ($imagefile, $imagebins);
  if (@todo && !$kiwimode && $preinstallimagedata && $Build::Features::preinstallimage) {
    my $imagesource;
    my $imagemetas;
    ($imagefile, $imagebins, $imagesource, $imagemetas) = getpreinstallimage($buildinfo, $dir, \@todo, \%bvls);
    if ($imagefile && $imagebins && $imagemetas) {
      $preinstallimagedata->{'imagename'} = $imagefile;
      $preinstallimagedata->{'imagebins'} = $imagebins;
      $preinstallimagedata->{'imagesource'} = $imagesource;
      $meta{$_} = 1 for keys %$imagemetas;
      # remove image packages from todo
      @todo = grep {!$imagebins->{$_}} @todo;
    }
  }
  $imagebins ||= {};

  for my $repo (@{$buildinfo->{'path'} || []}) {
    last if !@todo && !$kiwimode;
    my @args;
    my $ddir = $dir;
    my $dlbins;
    if ($kiwimode) {
      my $prpdir = "$repo->{'project'}/$repo->{'repository'}";
      $ddir = "$srcdir/repos/$prpdir";
      # Always try to get all binaries, because kiwi needs to decide which to take
      $dlbins = \@bdep;
    } else {
      $ddir = $dir;
      # get only missing packages
      $dlbins = \@todo;
    }
    my $nometa = 0;
    $nometa = 1 if $kiwimode || $repo->{'project'} ne $projid || $repo->{'repository'} ne $repoid;
    $nometa = 1 if $buildinfo->{'file'} eq '_preinstallimage';
    my $server = $repo->{'server'} || $buildinfo->{'reposerver'};
    my $got = getbinaries_cache($ddir, $server, $repo->{'project'}, $repo->{'repository'}, $buildinfo->{'arch'}, $nometa, $dlbins, $bvls{"$repo->{'project'}/$repo->{'repository'}/$buildinfo->{'arch'}"});
    for (keys %$got) {
      $done{$_} = $got->{$_}->{'name'};
      $meta{$_} = 1 if !$nometa && $got->{$_}->{'meta'};
    }
    @todo = grep {!$done{$_}} @todo;
    if ($kiwimode) {
      my @m;
      for my $n (keys %$got) {
	my $f = $got->{$n};
	if (!$bdep_notmeta{$n}) {
	  my $id = Build::queryhdrmd5("$ddir/$f->{'name'}") || "deaddeaddeaddeaddeaddeaddeaddead";
	  push @m, "$id  $repo->{'project'}/$repo->{'repository'}/$n";
	}
	if (!$buildinfo->{'syspath'} && !$bdep_noinstall{$n}) {
          if (!-e "$dir/$f->{'name'}") {
	    link_or_copy("$ddir/$f->{'name'}", "$dir/$f->{'name'}") || die("link_or_copy $ddir/$f->{'name'} $dir/$f->{'name'}: $!\n");
	  }
	}
      }
      push @meta, sort {substr($a, 34) cmp substr($b, 34)} @m;
    }
  }
  die("getbinaries: missing packages: @todo\n") if @todo;

  if (!$kiwimode) {
    # generate meta data
    # we carefully prune entries here to keep the memory usage down
    # so that coolo's image builds work
    # - do not prune entries that have one of our subpacks in the
    #   path as they are handled in a special way
    # - use the same order as the code in BSBuild
    my %mseen;
    my @subp = @{$buildinfo->{'subpack'} || []};
    my $subpackre = '';
    for (@subp) {
      $subpackre .= "|/\Q$_\E/";
    }
    if ($subpackre) {
      $subpackre = substr($subpackre, 1);
      $subpackre = qr/$subpackre/;
    }
    for my $dep (sort {"$a/" cmp "$b/"} map {$_->{'name'}} grep {!$_->{'notmeta'}} @{$buildinfo->{'bdep'} || []}) {
      my $m;
      $m = readstr("$dir/$dep.meta", 1) if $meta{$dep};
      if (!$m) {
	my $id = $imagebins->{$dep} || Build::queryhdrmd5("$dir/$done{$dep}") || "deaddeaddeaddeaddeaddeaddeaddead";
	push @meta, "$id  $dep";
      } else {
	chomp $m;
	my @m = split("\n", $m);
	$m[0] =~ s/  .*/  $dep/;
	push @meta, shift @m;
	if ($subpackre && "/$dep/" =~ /$subpackre/) {
	  s/  /  $dep\// for @m;
	  push @meta, @m;
	} else {
	  for (@m) {
	    next if $mseen{$_};
	    my $x = $_;
	    s/  /  $dep\//;
	    $mseen{$x} = 1 unless $subpackre && "$_/" =~ /$subpackre/;
	    push @meta, $_;
	  }
	}
      }
    }
    @meta = BSBuild::gen_meta(($buildinfo->{'verifymd5'} || $buildinfo->{'srcmd5'})."  $buildinfo->{'package'}", $buildinfo->{'subpack'} || [], @meta);
    shift @meta;	# strip srcinfo again
  }
  return @meta;
}

sub getoldpackages {
  my ($buildinfo, $odir) = @_;

  # get old package build for compare and diffing tools
  mkdir_p($odir) || die("mkdir_p $odir: $!\n");
  my $res = BSRPC::rpc({
    'uri' => "$buildinfo->{'reposerver'}/build/$buildinfo->{'project'}/$buildinfo->{'repository'}/$buildinfo->{'arch'}/$buildinfo->{'package'}",
    'directory' => $odir,
    'timeout' => $gettimeout,
    'receiver' => \&BSHTTP::cpio_receiver,
  }, undef, 'view=cpio', 'noajax=1');
  rmdir($odir);	# if not needed
}


sub patchkiwi {
  my ($buildinfo, $kiwifile, $meta) = @_;

  my $kiwi = readxml($kiwifile, $BSKiwiXML::kiwidesc);
  die("no instsource section in kiwi file\n") unless $kiwi->{'instsource'};
  my $repo_only;
  my @vars;
  my $tag_media;
  for my $productvar (@{$kiwi->{'instsource'}->{'productoptions'}->{'productvar'} || []}) {
    push @vars, $productvar;
    $repo_only = 1 if $productvar->{'name'} eq 'REPO_ONLY' and $productvar->{'_content'} eq 'true';
    if ($productvar->{'name'} eq 'MEDIUM_NAME') {
      my $mediumbase = $productvar->{'_content'};
      $mediumbase .= sprintf("-Build%04d", $buildinfo->{'bcnt'} || 0);
      pop @vars;
      push @vars, { 'name' => 'BUILD_ID', '_content' => $mediumbase };
      push @vars, { 'name' => 'MEDIUM_NAME', '_content' => "$mediumbase-Media" };
    } elsif ($productvar->{'name'} eq 'RUN_MEDIA_CHECK' && $productvar->{'_content'} eq 'true') {
      $tag_media = 1;
    }
  }
  $kiwi->{'instsource'}->{'productoptions'}->{'productvar'} = \@vars;

  for my $repopackages (@{$kiwi->{'instsource'}->{'repopackages'} || []}) {
    next unless grep {$_->{'name'} eq '*'} @{$repopackages->{'repopackage'} || []};
    # hey, a substitute all modifier!
    my @rp;
    my %allpkgs;
    for my $m (@$meta) {
      # md5  proj/rep/arch/pack/bin.arch
      my @s = split('/', $m);
      next unless $s[-1] =~ /^(.*)\.([^\.]*)$/;
      next if $2 eq 'src' || $2 eq 'nosrc';
      $allpkgs{$1} ||= {};
      $allpkgs{$1}->{$2} = 1;
    }
    for my $rp (@{$repopackages->{'repopackage'} || []}) {
      if ($rp->{'name'} ne '*') {
        push @rp, $rp;
        next;
      }

      for my $pkg (sort keys %allpkgs) {
        # exclude blind take of all debug packages. They will be taken
        # automatically if a configured debug medium exists.
        next if $pkg =~ /-debuginfo$/;
        next if $pkg =~ /-debugsource$/;
        next if $pkg =~ /-debuginfo-32bit$/;
        next if $pkg =~ /-debugsource-32bit$/;
        next if $pkg =~ /-debuginfo-64bit$/;
        next if $pkg =~ /-debugsource-64bit$/;
        next if $pkg =~ /-debuginfo-x86$/;
        next if $pkg =~ /-debugsource-x86$/;
        my @a;
        for my $availableArch (sort keys %{$allpkgs{$pkg}}){
          if ($availableArch eq "noarch") {
            for my $a (sort map { $_->{'ref'} } @{$kiwi->{'instsource'}->{'architectures'}->{'requiredarch'}}) {
              push @a, $a;
           }
          }else{
            push @a, $availableArch;
          }
        }
        push @rp, {'name' => $pkg, 'arch' => "".join(",", @a)};
      }
    }
    $repopackages->{'repopackage'} = \@rp;
  }
  writexml($kiwifile, undef, $kiwi, $BSKiwiXML::kiwidesc);
}

sub dobuild {
  my ($buildinfo) = @_;

  my $projid = $buildinfo->{'project'};
  my $packid = $buildinfo->{'package'};
  my $repoid = $buildinfo->{'repository'};
  my $arch = $buildinfo->{'arch'};
  my $helperarch = $buildinfo->{'hostarch'} || $arch;
  my $kiwimode;
  $kiwimode = 1 if $buildinfo->{'file'} =~ /\.kiwi$/;
  my $deltamode;
  $deltamode = 1 if $buildinfo->{'file'} eq '_delta';

  my $helper = '';
  /^\Q$helperarch\E:(.*)$/ && ($helper = $1) for @{$BSCando::cando{$hostarch}};

  my @lt = localtime(time());
  my $timestring = sprintf "%04d-%02d-%02d %02d:%02d:%02d", $lt[5] + 1900, $lt[4] + 1, @lt[3,2,1,0];
  
  print "$timestring: building '$packid' for project '$projid' repository '$repoid' arch '$arch'";
  print " using helper $helper" if $helper;
  print "\n";

  my $srcdir    = "$buildroot/.build-srcdir";
  my $pkgdir    = "$buildroot/.pkgs";
  my $oldpkgdir = "$buildroot/.build.oldpackages";

  if ($vm_tmpfs_mode) {
    my @targs;
    # Note that the xen/kvm vmdisk_rootsize is in MB
    push @targs, "mount", "-t", "tmpfs", "-osize=${vmdisk_rootsize}M", "none", $buildroot;
    qsystem(@targs) && die("mount tmpfs failed: $!\n");
  }

  unlink("$buildroot/.build.meta");
  rm_rf("$buildroot/.build.packages") if -d "$buildroot/.build.packages";
  rm_rf($srcdir) if -d $srcdir;
  # changed to cleandir so that pkgdir can be a symlink
  BSUtil::cleandir($pkgdir) if -d $pkgdir;
  rm_rf($oldpkgdir) if -d $oldpkgdir;

  my @meta;
  my $preinstallimagedata = {};
  print "fetching sources, ";
  mkdir($srcdir) || die("mkdir $srcdir: $!\n");
  if ($deltamode) {
    push @meta, getdeltasources($buildinfo, $srcdir);
    print "packages, ";
    getbinaries($buildinfo, $pkgdir, $srcdir, $preinstallimagedata);
    undef $oldpkgdir;
    my $spec = readstr("$statedir/worker/worker-deltagen.spec", 1) || readstr("worker-deltagen.spec");
    writestr("$srcdir/worker-deltagen.spec", undef, $spec);
    $buildinfo->{'file'} = 'worker-deltagen.spec';
  } else {
    push @meta, getsources($buildinfo, $srcdir);
    importbuild() unless defined &Build::queryhdrmd5;
    if (!$kiwimode && $Build::Features::preinstallimage && -s "$srcdir/$buildinfo->{'file'}") {
      # check if we may use a preinstallimage
      local *F;
      my $needbins;
      if (open(F, '<', "$srcdir/$buildinfo->{'file'}")) {
	while(<F>) {
	  chomp;
	  next unless /^#\s*needsbinariesforbuild\s*$/s;
	  undef $preinstallimagedata;	# can't use an image, sorry
	  last;
	}
	close F;
      }
    }
    print "packages, ";
    push @meta, getbinaries($buildinfo, $pkgdir, $srcdir, $preinstallimagedata);
  }
  $preinstallimagedata ||= {};

  writestr("$buildroot/.build.meta", undef, join("\n", @meta)."\n");

  getoldpackages($buildinfo, $oldpkgdir) if $oldpkgdir && !$kiwimode && $buildinfo->{'file'} ne 'preinstallimage';

  my @configpath;
  if ($kiwimode) {
    @configpath = map {"path=$_->{'project'}/$_->{'repository'}"} @{$buildinfo->{'syspath'} || $buildinfo->{'path'} || []};
    unshift @configpath, "path=$projid/$repoid" unless @configpath;
  }
  my $server = $buildinfo->{'srcserver'} || $srcserver;
  my $config = BSRPC::rpc("$server/getconfig", undef, "project=$projid", "repository=$repoid", @configpath);
  writestr("$buildroot/.build.config", undef, $config);

  my $release = $buildinfo->{'release'};
  my $obsinstance = $BSConfig::obsname || '';
  my $disturl = "obs://$obsinstance/$projid/$repoid/$buildinfo->{'srcmd5'}-$packid";

  my @args;
  push @args, $helper if $helper;

  # build rpmlist for build script
  my @rpmlist;
  my @bdep = @{$buildinfo->{'bdep'} || []};
  my $imagebins = $preinstallimagedata->{'imagebins'} || {};
  for my $bdep (@bdep) {
    next if $bdep->{'package'} || $bdep->{'noinstall'} || ($bdep->{'repoarch'} && $bdep->{'repoarch'} eq 'src');
    my $bin = $bdep->{'name'};
    if ($imagebins->{$bin}) {
	push @rpmlist, "$bin preinstallimage";
	next;
    }
    for my $osuf (@binsufs, '__notfound') {
      die("missing package: $bin\n") if $osuf eq '__notfound';
      next unless -e "$pkgdir/$bin.$osuf";
      push @rpmlist, "$bin $pkgdir/$bin.$osuf";
      last;
    }
  }
  push @rpmlist, "preinstallimage: $pkgdir/$preinstallimagedata->{'imagename'}" if $preinstallimagedata->{'imagename'};
  push @rpmlist, "preinstallimagesource: $preinstallimagedata->{'imagesource'}" if $preinstallimagedata->{'imagesource'};
  push @rpmlist, "localkiwi $localkiwi/localkiwi.rpm" if $localkiwi && -e "$localkiwi/localkiwi.rpm";
  push @rpmlist, "preinstall: ".join(' ', map {$_->{'name'}} grep {$_->{'preinstall'}} @bdep);
  push @rpmlist, "vminstall: ".join(' ', map {$_->{'name'}} grep {$_->{'vminstall'}} @bdep);
  push @rpmlist, "runscripts: ".join(' ', map {$_->{'name'}} grep {$_->{'runscripts'}} @bdep);
  if (grep {$_->{'cbpreinstall'}} @bdep) {
    push @rpmlist, "cbpreinstall: ".join(' ', map {$_->{'name'}} grep {$_->{'cbpreinstall'}} @bdep);
  }
  if (grep {$_->{'cbinstall'}} @bdep) {
    push @rpmlist, "cbinstall: ".join(' ', map {$_->{'name'}} grep {$_->{'cbinstall'}} @bdep);
  }
  writestr("$buildroot/.build.rpmlist", undef, join("\n", @rpmlist)."\n");

  print "building...\n";

  if ($kiwimode && $buildinfo->{'imagetype'} && $buildinfo->{'imagetype'}->[0] eq 'product') {
    patchkiwi($buildinfo, "$srcdir/$buildinfo->{'file'}", \@meta);
  }

  push @args, "$statedir/build/build";
  if ($vm =~ /(xen|kvm)/) {
    # add a bit delay - not all workers should start at the same time after idle.
    sleep (int(rand(5)) + 1); 
    mkdir("$buildroot/.mount") unless -d "$buildroot/.mount";
    push @args, '--root', "$buildroot/.mount";
    push @args, $vm, "$vm_root";
    push @args, '--swap', "$vm_swap";
    my $vmmemory = readstr("$buildroot/memory", 1);
    $vmmemory = $vm_memory if $vm_memory;
    push @args, '--memory', $vmmemory if $vmmemory;
    push @args, '--vm-kernel', $vm_kernel if $vm_kernel;
    push @args, '--vm-initrd', $vm_initrd if $vm_initrd;
    push @args, '--vmdisk-rootsize', $vmdisk_rootsize if $vmdisk_rootsize;
    push @args, '--vmdisk-swapsize', $vmdisk_swapsize if $vmdisk_swapsize;
    push @args, '--vmdisk-filesystem', $vmdisk_filesystem if $vmdisk_filesystem;
    # mount options require quoting due to arguments which might start with "-o ..."
    push @args, '--vmdisk-mount-options', "\"$vmdisk_mount_options\"" if $vmdisk_mount_options;
    push @args, '--hugetlbfs', "\"$hugetlbfs\"" if $hugetlbfs;
  } elsif ($vm =~ /lxc/) {
    push @args, '--root', $buildroot;
    push @args, '--vm-type', $vm;
  } else {
    push @args, '--root', $buildroot;
  }
  push @args, '--clean';
  push @args, '--changelog';
  push @args, '--oldpackages', $oldpkgdir if $oldpkgdir && -d $oldpkgdir;
  push @args, '--norootforbuild' unless $BSConfig::norootexceptions && grep {"$projid/$packid" =~ /^$_$/} keys %$BSConfig::norootexceptions;
  push @args, '--baselibs-internal';
  push @args, '--lint';
  push @args, '--dist', "$buildroot/.build.config";
  push @args, '--rpmlist', "$buildroot/.build.rpmlist";
  push @args, '--logfile', "$buildroot/.build.log";
  push @args, '--release', "$release" if defined $release;
  push @args, '--debug' if $buildinfo->{'debuginfo'};
  push @args, '--arch', $arch;
  push @args, '--jobs', $jobs if $jobs;
  push @args, '--reason', "Building $packid for project '$projid' repository '$repoid' arch '$arch' srcmd5 '$buildinfo->{'srcmd5'}'";
  push @args, '--disturl', $disturl;
  push @args, '--linksources' if $localkiwi;
  push @args, '--signdummy' if $kiwimode && (!$localkiwi || ! -e "$localkiwi/localkiwi.rpm") && $buildinfo->{'imagetype'} && $buildinfo->{'imagetype'}->[0] eq 'product' && -e "$statedir/build/signdummy";
  push @args, "$srcdir/$buildinfo->{'file'}";
  qsystem(@args);

  print "\n";
  print "$timestring: build finished '$packid' for project '$projid' repository '$repoid' arch '$arch'";
  print " using helper $helper" if $helper;
  print "\n";

  my $ret = $?;
  if ($ret == 512) { # system exit code 2 maps to 2 * 256
    if (($buildinfo->{'reason'} || '') eq "rebuild counter sync") {
      $ret = 0;
    } else {
      return 2;
    }
  } elsif ($ret == 768) {
    return 3;
  }
  if ($ret) {
    return 1;
  }
  if (! -s "$buildroot/.build.log") {
    print "build succeeded, but no logfile?\n";
    return 1;
  }

  if ($vm =~ /(xen|kvm)/) {
    rm_rf("$buildroot/.build.packages");
    if (! -d "$buildroot/.mount/.build.packages") {
      # old style, call extractbuild
      print "extracting built packages...\n";
      @args = ();
      push @args, "$statedir/build/extractbuild";
      push @args, '--root', "$vm_root";
      push @args, '--swap', "$vm_swap";
      if (system(@args)) {
        die("extractbuild failed\n");
      }
      mkdir_p("$buildroot/.build.packages");
      if (system("cd $buildroot/.build.packages && cpio --extract --no-absolute-filenames -v < $vm_swap")) {
	die("cpio extract failed\n");
      }
    } else {
      # new style, build already did extractbuild for us
      if(!rename("$buildroot/.mount/.build.packages", "$buildroot/.build.packages")) {
        print "final rename failed: $!";
        return 1;
      }
    }
    # XXX: extracted cpio is flat but code below expects those directories...
    symlink('.', "$buildroot/.build.packages/SRPMS");
    symlink('.', "$buildroot/.build.packages/DEBS");
    symlink('.', "$buildroot/.build.packages/KIWI");
  }

  return 0;
}

sub buildkiwitree {
  my ($destfile, @todo) = @_;
  my %tree;
  @todo = map {($_, '')} @todo;
  while (@todo) {
    my $dir = shift @todo;
    my $sdir = shift @todo;
    for my $f (sort(ls($dir))) {
      my $sf = "$sdir$f";
      $f = "$dir/$f";
      next if $tree{$sf};
      if (-l $f) {
	my $sl = readlink($f);
	die("readlink $f: $!\n") unless defined $sl;
	$tree{$sf} = "l ". BSRPC::urlencode($sf) . " " . BSRPC::urlencode($sl);
      } elsif (-d $f) {
	$tree{$sf} = "d ". BSRPC::urlencode($sf);
	push @todo, $f, "$sf/";
      } elsif (-f $f) {
        my $leadsigmd5 = '';
	Build::queryhdrmd5($f, \$leadsigmd5) if $f =~ /\.rpm$/;
	if ($leadsigmd5) {
	  $tree{$sf} = "f ". BSRPC::urlencode($sf) . " " . $leadsigmd5;
	} else {
	  $tree{$sf} = "f ". BSRPC::urlencode($sf);
	}
      }
    }
  }
  local *F;
  open(F, '>', $destfile) || die("$destfile: $!\n");
  for (sort keys %tree) {
    print F "$tree{$_}\n";
  }
  close(F) || die("close $destfile: $!\n");
}

if (defined($oneshot)) {
  $oneshot = time() + $oneshot;
}

# better safe than sorry...
chdir($statedir) || die("$statedir: $!\n");


BSServer::deamonize(@ARGV) unless $oneshot;
$SIG{'PIPE'} = 'IGNORE';

# calculate code meta md5
my $workercode = codemd5('worker');
my $buildcode = codemd5('build');
$| = 1;
print "starting worker $workercode build $buildcode\n";

open(RUNLOCK, '>>', "$statedir/lock") || die("$statedir/lock: $!");
flock(RUNLOCK, LOCK_EX | LOCK_NB) || die("worker is already running on $statedir!\n");
utime undef, undef, "$statedir/lock";

# we always start idle
lockstate();
unlink("$statedir/job");
unlink("$buildroot/.build.log");
commitstate({'state' => 'idle'});

# start server process...
if ($port) {
  BSServer::serveropen($port);
} else {
  BSServer::serveropen(\$port);
}
mkdir($buildroot) unless -d $buildroot;
send_state('idle', $port, $hostarch);

my $idlecnt = 0;
my $rekillcnt = 0;

my $conf = {
  'timeout' => 10,
};
while (!BSServer::server($conf)) {
  # timeout handler, called every 10 seconds
  my $state = readxml("$statedir/state", $BSXML::workerstate, 1);
  next unless $state;

  if ($state->{'state'} eq 'idle') {
    if ($oneshot && time() > $oneshot) {
      send_state('exit', $port, $hostarch);
      print "exiting.\n";
      exit(0);
    }
    $idlecnt++;
    if ($idlecnt % 30 == 0) {
      # send idle message every 5 minutes in case the server was down
      $idlecnt = 0;
      send_state('idle', $port, $hostarch) if $state->{'state'} eq 'idle';
    }
  } else {
    $idlecnt = 0;
  }

  if ($state->{'state'} eq 'exit') {
    $state = lockstate();
    if ($state->{'state'} eq 'exit') {
      $state = {'state' => 'idle'};
      commitstate($state);
      send_state('exit', $port, $hostarch);
      exit(0);
    }
    unlockstate();
  }

  if ($state->{'state'} eq 'rebooting') {
    chdir("$statedir/worker") || die("$statedir/worker: $!");
    close RUNLOCK;
    exec("./bs_worker", @saveargv);
    die("$statedir/worker/bs_worker: $!\n");	# oops
  }

  if ($state->{'state'} eq 'killed' || $state->{'state'} eq 'discarded') {
    $rekillcnt++;
    if ($state->{'state'} eq 'discarded' && $state->{'nextstate'}) {
      # signal early that we're going down
      send_state('exit', $port, $hostarch);
      $rekillcnt = 12;
    }
    if ($rekillcnt % 12 == 0) {
      # re-kill after 2 minutes, maybe build is stuck somewhere
      $rekillcnt = 0;
      $state = lockstate();
      if ($state->{'state'} eq 'killed' || $state->{'state'} eq 'discarded') {
        kill_job();
      }
      unlockstate();
    }
  } else {
    $rekillcnt = 0;
  }

  next unless $state->{'state'} eq 'building';

  my $locked = -1;
  while ($locked++ < 1) {
    $state = lockstate() if $locked == 1;
    last if $state->{'state'} ne 'building';
    my $ct = time();
    my @s = stat("$buildroot/.build.log");
    next unless @s;
    if ($s[7] > $buildlog_maxsize) {
      next unless $locked;
      if (!kill_job()) {
        warn("could not kill job\n");
        last;
      }
      trunc_logfile("$buildroot/.build.log");
      $state->{'state'} = 'killed';
      commitstate($state);
      $locked = 0;
    } elsif ($ct - $s[9] > $buildlog_maxidle) {
      next unless $locked;
      if (!kill_job()) {
        warn("could not kill job\n");
        last;
      }
      local *F;
      if (open(F, '>>', "$buildroot/.build.log")) {
	print F "\n\nJob seems to be stuck here, killed.\n";
	close F;
      }
      $state->{'state'} = 'killed';
      commitstate($state);
      $locked = 0;
    }
    last;
  }
  unlockstate() if $locked;
}
close RUNLOCK;

my $req = BSServer::readrequest();
my $path = $req->{'path'};
my $multies;
$multies = {'file' => undef} if $path eq '/kiwitree';
my $cgi = BSServer::parse_cgi($req, $multies);
if ($path eq '/info') {
  # check state?
  my $state = readxml("$statedir/state", $BSXML::workerstate, 1);
  if ($cgi->{'jobid'}) {
    die("not building a job\n") if $state->{'state'} ne 'building';
    die("building a different job\n") if $cgi->{'jobid'} ne $state->{'jobid'};
  }
  my $info;
  if ($state->{'state'} eq 'building') {
    $info = readstr("$statedir/job");
  } else {
    $info = "<buildinfo>\n  <error>".$state->{'state'}."</error>\n</buildinfo>\n";
  }
  BSServer::reply($info, 'Content-Type: text/xml');
  exit(0);
} elsif ($path eq '/worker') {
  my $state = readxml("$statedir/state", $BSXML::workerstate, 1);
  if ($cgi->{'jobid'}) {
    die("not building a job\n") if $state->{'state'} ne 'building';
    die("building a different job\n") if $cgi->{'jobid'} ne $state->{'jobid'};
  }
  my $info = {};
  $info->{'hostarch'} = $hostarch;
  $info->{'port'} = $port;
  $info->{'workerid'} = $workerid if defined $workerid;
  # $info->{'job'} = $state->{'job'};
  # $info->{'arch'} = $state->{'arch'};
  $info = XMLout($BSXML::worker, $info);
  BSServer::reply($info, 'Content-Type: text/xml');
  exit(0);
} elsif ($path eq '/kiwitree') {
  my $state = readxml("$statedir/state", $BSXML::workerstate, 1);
  if ($cgi->{'jobid'}) {
    die("not building a job\n") if $state->{'state'} ne 'building';
    die("building a different job\n") if $cgi->{'jobid'} ne $state->{'jobid'};
  }
  my $kiwitreefile = "$buildroot/.build.packages/.kiwitree";
  die("kiwitreefile does not exist\n") unless -e $kiwitreefile;
  my @send;
  for my $f (@{$cgi->{'file'} || []}) {
    die("$f: $!\n") unless -f "$buildroot/.build.packages/KIWI/$f";
    push @send, { 'name' => $f, 'filename' => "$buildroot/.build.packages/KIWI/$f" };
  }
  BSServer::reply_cpio(\@send);
  exit(0);
} elsif ($path eq '/logfile') {
  my $state = readxml("$statedir/state", $BSXML::workerstate, 1);
  die("not building\n") if $state->{'state'} ne 'building';
  die("building a different job\n") if $cgi->{'jobid'} && $cgi->{'jobid'} ne $state->{'jobid'};
  if ($cgi->{'view'} && $cgi->{'view'} eq 'entry') {
    my @s = stat("$buildroot/.build.log");
    die("$buildroot/.build.log: $!\n") unless @s;
    my $xml = "<directory>\n  <entry name=\"_log\" size=\"$s[7]\" mtime=\"$s[9]\" />\n</directory>\n";
    BSServer::reply($xml, 'Content-Type: text/xml');
    exit(0);
  }
  stream_logfile($cgi->{'nostream'}, $cgi->{'start'}, $cgi->{'end'});
  exit(0);
} elsif ($path eq '/kill' || $path eq '/discard') {
  my $state = lockstate();
  die("not building\n") if $state->{'state'} ne 'building';
  die("building a different job\n") if $cgi->{'jobid'} && $cgi->{'jobid'} ne $state->{'jobid'};
  if (!kill_job()) {
    die("could not kill job\n");
  }
  local *F;
  if (open(F, '>>', "$buildroot/.build.log")) {
    if ($path eq '/kill') {
      print F "\n\nKilled Job\n";
    } else {
      print F "\n\nDiscarded Job\n";
    }
    close F;
  }
  if ($path eq '/kill') {
    $state->{'state'} = 'killed';
    commitstate($state);
    BSServer::reply("<status=\"ok\" />\n", 'Content-Type: text/xml');
  } else {
    $state->{'state'} = 'discarded';
    commitstate($state);
    BSServer::reply("<status=\"ok\" />\n", 'Content-Type: text/xml');
  }
  exit(0);
} elsif ($path ne '/build' || $req->{'action'} ne 'PUT') {
  die("unknown request: $path\n");
}

# Check for XEN daemon database leak
if ($vm eq "--xen" && $xenstore_maxsize && 0 + (-s '/var/lib/xenstored/tdb') > $xenstore_maxsize) {
  die("xenstore too big:".(-s '/var/lib/xenstored/tdb')."\n");
}

my $state = lockstate();
if ($cgi->{'workercode'} && $cgi->{'port'} && $cgi->{'workercode'} ne $workercode && !$noworkercheck) {
  $state->{'state'} = 'rebooting';
  my $peer = "${BSServer::peer}:$cgi->{'port'}";
  $workercode = getcode('worker', "http://$peer/getworkercode");
  if (!$workercode) {
    $state->{'state'} = 'broken';	# eek
  } else {
    print "activating new worker code $workercode\n";
  }
  commitstate($state);
  die("rebooting...\n");
}

die("I am not idle!\n") unless $state->{'state'} eq 'idle';

BSServer::read_file('job.new');
my $infoxml = readstr('job.new');
die("bad job xml data\n") unless $infoxml =~ /<.*?>/s;
my $buildinfo = XMLin($BSXML::buildinfo, $infoxml);
my $jobid = $cgi->{'jobid'};
$jobid ||= Digest::MD5::md5_hex($infoxml);

$buildinfo->{'jobid'} = $jobid;

# old buildinfos missed some entries
if (@{$buildinfo->{'path'} || []}) {
  $buildinfo->{'project'} ||= $buildinfo->{'path'}->[0]->{'project'};
  $buildinfo->{'repository'} ||= $buildinfo->{'path'}->[0]->{'repository'};
  $buildinfo->{'reposerver'} ||= $buildinfo->{'path'}->[0]->{'server'};
}

if ($localkiwi) {
  # make sure this is the right job for us
  die("not a kiwi job\n") unless $buildinfo->{'file'} =~ /\.kiwi$/;
  die("not a kiwi product job\n") unless $buildinfo->{'imagetype'} && $buildinfo->{'imagetype'}->[0] eq 'product';
}

$buildcode = codemd5('build');
if (!$nobuildcodecheck && $cgi->{'buildcode'} && $cgi->{'port'} && $cgi->{'buildcode'} ne $buildcode) {
  print "fetching new buildcode $cgi->{'buildcode'}, mine was $buildcode\n";
  my $peer = "${BSServer::peer}:$cgi->{'port'}";
  $buildcode = getcode('build', "http://$peer/getbuildcode");
  die("could not update build code\n") unless $buildcode;
}

rename('job.new', 'job') || die("rename job.new job: $!\n");

if ($hostcheck) {
  my $server = $buildinfo->{'srcserver'} || $srcserver;
  if (system($hostcheck, '--srcserver', $server, "$statedir/job", 'precheck', $buildroot)) {
    unlink('job');
    die("400 cannot build this package\n");
  }
}

if ($testmode) {
  BSServer::reply("<status code=\"failed\">\n  <details>testmode activated</details>\n</status>\n", 'Status: 400 Testmode', 'Content-Type: text/xml');
} else {
  BSServer::reply("<status code=\"ok\">\n  <details>so much work, so little time...</details>\n</status>\n", 'Content-Type: text/xml');
}
print "got job, run build...\n";
delete $SIG{'__DIE__'};
unlink("$buildroot/.build.meta");
unlink("$buildroot/.build.packages");
unlink("$buildroot/.build.log");
writestr("$buildroot/.build.log", undef, '');

$state->{'state'} = 'building';
$state->{'jobid'} = $jobid;
commitstate($state);

send_state('building', $port, $hostarch, $buildinfo->{'reposerver'});

my $ex;
eval {
  $ex = dobuild($buildinfo);
};
if ($@) {
  local *F;
  if (open(F, '>>', "$buildroot/.build.log")) {
    print F $@;
    close(F);
  }
  print "$@";
  $ex = 1;
}

# build is done, send back result
$state = lockstate();

if ($state->{'state'} eq 'discarded') {
  # our poor job is no longer needed
  print "build discarded...\n";
  unlink("$buildroot/.build.log");
  unlink("$buildroot/job");
  cleanup_job();

  if ($state->{'nextstate'}) {
    $state = {'state' => $state->{'nextstate'}};
    commitstate($state);
    exit(0);
  }
  $state = {'state' => 'idle'};
  commitstate($state);
  exit(0) if $oneshot && time() > $oneshot;
  send_state('idle', $port, $hostarch);
  exit(0);
}

if ($state->{'state'} ne 'building') {
  # something is wrong, consider job bad
  $ex = 1;
}

if (! -s "$buildroot/.build.log") {
  eval {
    if (defined($workerid)) {
      writestr("$buildroot/.build.log", undef, "build on $workerid did not create a logfile\n");
    } else {
      writestr("$buildroot/.build.log", undef, "build did not create a logfile\n");
    }
  };
  $ex = 3;
}

if ($hostcheck) {
  print "running post-build host check\n";
  my $server = $buildinfo->{'srcserver'} || $srcserver;
  if (system($hostcheck, '--srcserver', $server, "$statedir/job", $ex ? 'failed' : 'succeeded', "$buildroot/.build.log")) {
    print "post-build host check failed\n";
    $ex = 3;
  }
}

my @send;
my $kiwitree;
if ($ex == 0 && $buildinfo->{'reason'} ne "rebuild counter sync" && -f "$buildroot/.build.packages/same_result_marker") {
  $ex = 2;
}
if ($ex == 0) {
  my @d;
  push @d, map {"RPMS/$_"} sort(ls("$buildroot/.build.packages/RPMS"));
  push @d, 'SRPMS';
  @d = ('DEBS') if $buildinfo->{'file'} =~ /\.dsc$/;
  @d = ('ARCHPKGS') if $buildinfo->{'file'} eq 'PKGBUILD';
  @d = ('KIWI') if $buildinfo->{'file'} =~ /\.kiwi$/;
  push @d, 'OTHER';
  for my $d ('.', @d) {
    my @files = sort(ls("$buildroot/.build.packages/$d"));
    @files = grep {$_ ne 'same_result_marker' && $_ ne '.kiwitree'} @files;
    if ($localkiwi && ! -d "$localkiwi/jobs/$buildinfo->{'arch'}" && $d eq 'KIWI') {
      $kiwitree = 1 if grep {-d "$buildroot/.build.packages/$d/$_"} @files;
      next if $kiwitree;
    }
    @files = grep {-f "$buildroot/.build.packages/$d/$_"} @files;
    push @send, map {"$buildroot/.build.packages/$d/$_"} @files;
  }
  @send = map {{name => (split('/', $_))[-1], filename => $_}} @send;
  if ($kiwitree) {
    my $kiwitreefile = "$buildroot/.build.packages/.kiwitree";
    eval {
      die(".kiwitree already exists\n") if -e $kiwitreefile;
      buildkiwitree($kiwitreefile , "$buildroot/.build.packages/KIWI");
      push @send, { name => '.kiwitree', filename => $kiwitreefile };
    };
    if ($@) {
      print "could not create kiwitree: $@\n";
      BSUtil::appendstr("$buildroot/.build.log", "\ncould not create kiwitree: $@\n");
      $ex = 1;
    }
  }
  if (!@send && !$localkiwi) {
    print "build did not create anything to send back!\n";
    $ex = 1;
  }
}
my $code;
if (!$ex) {
  print "build succeeded, send everything back...\n";
  $code = 'succeeded';
} elsif ($ex == 2) {
  print "build succeeded, but does not differ from old build result...\n";
  $code = 'unchanged';
} elsif ($ex == 3) {
  print "build failed, marked as bad build host...\n";
  $code = 'badhost';
  # try to be clever and avoid endless builds of people who broken their build enviroment
  if (open(LL, '<', "$buildroot/.build.log")) {
    my $data = '';
    sysseek(LL, -10240, 2);
    sysread(LL, $data, 10240);
    close LL;
    if ($data =~ /^mount: error while loading shared libraries:/m) {
      print "Wait ... spotted failed build. Host is good.\n";
      $code = 'failed';
    }
  }
} else {
  print "build failed, send back logfile...\n";
  $code = 'failed';
}
push @send, {name => 'meta', filename => "$buildroot/.build.meta"} if -e "$buildroot/.build.meta";
push @send, {name => 'logfile', filename => "$buildroot/.build.log"};

if (!$testmode) {
  if ($localkiwi && -d "$localkiwi/jobs/$buildinfo->{'arch'}") {
    # local delivery hack...
    my @s = stat(_);
    my ($localkiwi_uid, $localkiwi_gid) = ($s[4], $s[5]);
    my $jobdir = "$localkiwi/jobs/$buildinfo->{'arch'}/$buildinfo->{'job'}:dir";
    mkdir_p($jobdir);
    chown($localkiwi_uid, $localkiwi_gid, $jobdir);
    BSUtil::cleandir($jobdir);
    for my $f (ls("$buildroot/.build.packages/KIWI")) {
      system('chown', '-h', '-R', '--reference', "$jobdir", "$buildroot/.build.packages/KIWI/$f");
      rename("$buildroot/.build.packages/KIWI/$f", "$jobdir/$f") || die("rename $buildroot/.build.packages/KIWI/$f $jobdir/$f: $!\n");
    }
    chown($localkiwi_uid, $localkiwi_gid, "$buildroot/.build.log");
    chown($localkiwi_uid, $localkiwi_gid, "$buildroot/.build.meta");
    rename("$buildroot/.build.log", "$jobdir/logfile");
    rename("$buildroot/.build.meta", "$jobdir/meta");
    @send = ();
  }
  my $param = {
    uri => "$buildinfo->{'reposerver'}/putjob",
    request => 'POST',
    headers => [ 'Content-Type: application/x-cpio' ],
    chunked => 1,
    data => \&BSHTTP::cpio_sender,
    cpiofiles => \@send,
  };
  my $now = time();
  my @args = ("job=$buildinfo->{'job'}", "arch=$buildinfo->{'arch'}", "jobid=$jobid", "code=$code", "now=$now");
  push @args, "workerid=$workerid" if defined $workerid;
  push @args, "kiwitree=1" if $kiwitree && $code eq 'succeeded';
  if ($code eq 'badhost') {
    # don't transmit anything in the badhost case
    $param = {
      uri => "$buildinfo->{'reposerver'}/putjob",
      request => 'POST',
    };
  }
  eval {
    my $res = BSRPC::rpc($param, undef, @args);
  };
  if ($@) {
    print "rpc failed: $@\nsleeping one minute just in case...\n";
    sleep(60);
  } else {
    print "sent, all done...\n";
  }
} else {
  print "testmode, not sending anything\n";
  print Dumper(\@send);
}

unlink("$buildroot/.build.log");
unlink("$buildroot/job");
print "\n";
cleanup_job();

if ($state->{'nextstate'}) {
  $state = {'state' => $state->{'nextstate'}};
  commitstate($state);
  exit(0);
}
$state = {'state' => 'idle'};
commitstate($state);

exit(0) if $oneshot && time() > $oneshot;
send_state('idle', $port, $hostarch);

exit(0);