diff --git a/dist/Makefile b/dist/Makefile index c8d1c5a68fe..846be70a8fe 100644 --- a/dist/Makefile +++ b/dist/Makefile @@ -1,6 +1,6 @@ include ../Makefile.include -INIT_SCRIPTS := obssrcserver obsrepserver obsscheduler obsworker obspublisher obsdispatcher obssigner obswarden obsapidelayed obsapisetup obsstoragesetup obsservice obsdodup obsservicedispatch +INIT_SCRIPTS := obssrcserver obsrepserver obsscheduler obsworker obspublisher obsdispatcher obssigner obswarden obsapidelayed obsapisetup obsstoragesetup obsservice obsdodup obsservicedispatch obsclouduploadserver obsclouduploadworker LOGROTATE_CONFIGS := obs-api obs-server obs-source_service OBS_BIN_SCRIPTS := obs_productconvert OBS_SBIN_SCRIPTS := obs_admin obs_serverstatus diff --git a/dist/obs-server.spec b/dist/obs-server.spec index 8a887a95c14..7af7a043da9 100644 --- a/dist/obs-server.spec +++ b/dist/obs-server.spec @@ -409,7 +409,7 @@ getent passwd obsrun >/dev/null || \ exit 0 %preun -%stop_on_removal obssrcserver obsrepserver obsdispatcher obsscheduler obspublisher obswarden obssigner obsdodup obsservicedispatch obsservice +%stop_on_removal obssrcserver obsrepserver obsdispatcher obsscheduler obspublisher obswarden obssigner obsdodup obsservicedispatch obsservice obsclouduploadworker obsclouduploadserver %service_del_preun obsdeltastore @@ -418,9 +418,9 @@ exit 0 %post %if 0%{?suse_version} >= 1315 -%reload_on_update obssrcserver obsrepserver obsdispatcher obspublisher obswarden obssigner obsdodup obsservicedispatch obsservice +%reload_on_update obssrcserver obsrepserver obsdispatcher obspublisher obswarden obssigner obsdodup obsservicedispatch obsservice obsclouduploadworker obsclouduploadserver %else -%restart_on_update obssrcserver obsrepserver obsdispatcher obspublisher obswarden obssigner obsdodup obsservicedispatch obsservice +%restart_on_update obssrcserver obsrepserver obsdispatcher obspublisher obswarden obssigner obsdodup obsservicedispatch obsservice obsclouduploadworker obsclouduploadserver %endif # systemd kills the init script executing the reload first on reload.... %restart_on_update obsscheduler @@ -506,6 +506,8 @@ chown %{apache_user}:%{apache_group} /srv/www/obs/api/log/production.log %{_unitdir}/obsdeltastore.service /etc/init.d/obsservicedispatch /etc/init.d/obssigner +/etc/init.d/obsclouduploadworker +/etc/init.d/obsclouduploadserver /usr/sbin/obs_admin /usr/sbin/obs_serverstatus /usr/sbin/rcobsdispatcher @@ -518,6 +520,8 @@ chown %{apache_user}:%{apache_group} /srv/www/obs/api/log/production.log /usr/sbin/rcobsdeltastore /usr/sbin/rcobsservicedispatch /usr/sbin/rcobssigner +/usr/sbin/rcobsclouduploadworker +/usr/sbin/rcobsclouduploadserver /usr/lib/obs/server/plugins /usr/lib/obs/server/BSDispatcher /usr/lib/obs/server/BSRepServer @@ -550,6 +554,7 @@ chown %{apache_user}:%{apache_group} /srv/www/obs/api/log/production.log /usr/lib/obs/server/bs_signer /usr/lib/obs/server/bs_warden /usr/lib/obs/server/bs_clouduploadserver +/usr/lib/obs/server/bs_clouduploadworker /usr/lib/obs/server/worker /usr/lib/obs/server/worker-deltagen.spec %config(noreplace) /usr/lib/obs/server/BSConfig.pm diff --git a/dist/obsclouduploadserver b/dist/obsclouduploadserver new file mode 100755 index 00000000000..7176cab8d3b --- /dev/null +++ b/dist/obsclouduploadserver @@ -0,0 +1,94 @@ +#! /bin/sh +# Copyright (c) 2007, Novell Inc. +# +# Author: adrian@suse.de +# +# /etc/init.d/obsclouduploadserver +# and its symbolic link +# /usr/sbin/rcobsclouduploadserver +# +### BEGIN INIT INFO +# Provides: obsclouduploadserver +# Required-Start: $time $syslog +# Required-Stop: $null +# Should-Start: $none +# Should-Stop: $none +# Default-Start: 3 5 +# Default-Stop: 0 1 2 4 6 +# Description: open build service cloup upload server +### END INIT INFO + +. /etc/rc.status + +. /etc/sysconfig/obs-server + +# Determine the base and follow a runlevel link name. +base=${0##*/} +link=${base#*[SK][0-9][0-9]} + +if [ -z "$OBS_RUN_DIR" ]; then + OBS_RUN_DIR="/srv/obs/run" +fi + +if [ -z "$OBS_LOG_DIR" ]; then + OBS_LOG_DIR="/srv/obs/log" +fi + +if [ -n "$OBS_BACKENDCODE_DIR" ]; then + obsdir="$OBS_BACKENDCODE_DIR" +else + obsdir=/usr/lib/obs/server +fi +rundir="$OBS_RUN_DIR" +logdir="$OBS_LOG_DIR" + +rundir_perm() { + # make sure rundir is group writable + test "$(stat -c "%A" "$rundir" | cut -c6)" = "-" && chmod 0775 "$rundir" +} + +rc_reset +case "$1" in + start) + echo -n "Initializing obsclouduploadserver" + mkdir -p "$rundir" "$logdir" + rundir_perm + chown obsrun:obsrun "$logdir" "$rundir" + startproc -f -l "$logdir"/clouduploadserver.log "$obsdir"/bs_clouduploadserver + rc_status -v + ;; + stop) + echo -n "Shutting down obsservice" + "$obsdir"/bs_clouduploadserver --stop + rc_status -v + ;; + restart) + ## If first returns OK call the second, if first or + ## second command fails, set echo return value. + rundir_perm + "$obsdir"/bs_clouduploadserver --restart + rc_status + ;; + try-restart|reload) + $0 status + if test $? = 0; then + rundir_perm + "$obsdir"/bs_clouduploadserver --restart + else + rc_reset # Not running is not a failure. + fi + # Remember status and be quiet + rc_status + ;; + status) + echo -n "Checking for obsclouduploadserver and running processes: " + "$obsdir"/bs_serverstatus "$OBS_RUN_DIR"/bs_clouduploadserver.status + checkproc "$obsdir"/bs_clouduploadserver + rc_status -v + ;; + *) + echo "Usage: $0 {start|stop|status|try-restart|restart|reload}" + exit 1 + ;; +esac +rc_exit diff --git a/dist/obsclouduploadworker b/dist/obsclouduploadworker new file mode 100755 index 00000000000..29ee50b9245 --- /dev/null +++ b/dist/obsclouduploadworker @@ -0,0 +1,93 @@ +#! /bin/sh +# Copyright (c) 2007, Novell Inc. +# +# Author: adrian@suse.de +# +# /etc/init.d/obsclouduploadworker +# and its symbolic link +# /usr/sbin/rcobsclouduploadworker +# +### BEGIN INIT INFO +# Provides: obsclouduploadworker +# Required-Start: $time $syslog +# Required-Stop: $null +# Should-Start: $none +# Should-Stop: $none +# Default-Start: 3 5 +# Default-Stop: 0 1 2 4 6 +# Description: open build service cloup upload worker +### END INIT INFO + +. /etc/rc.status + +. /etc/sysconfig/obs-server + +# Determine the base and follow a runlevel link name. +base=${0##*/} +link=${base#*[SK][0-9][0-9]} + +if [ -z "$OBS_RUN_DIR" ]; then + OBS_RUN_DIR="/srv/obs/run" +fi + +if [ -z "$OBS_LOG_DIR" ]; then + OBS_LOG_DIR="/srv/obs/log" +fi + +if [ -n "$OBS_BACKENDCODE_DIR" ]; then + obsdir="$OBS_BACKENDCODE_DIR" +else + obsdir=/usr/lib/obs/server +fi +rundir="$OBS_RUN_DIR" +logdir="$OBS_LOG_DIR" + +rundir_perm() { + # make sure rundir is group writable + test "$(stat -c "%A" "$rundir" | cut -c6)" = "-" && chmod 0775 "$rundir" +} + +rc_reset +case "$1" in + start) + echo -n "Initializing obsclouduploadworker" + mkdir -p "$rundir" "$logdir" + rundir_perm + chown obsrun:obsrun "$logdir" "$rundir" + startproc -f -l "$logdir"/clouduploadworker.log "$obsdir"/bs_clouduploadworker + rc_status -v + ;; + stop) + echo -n "Shutting down obsclouduploadworker" + "$obsdir"/bs_clouduploadworker --stop + rc_status -v + ;; + restart) + ## If first returns OK call the second, if first or + ## second command fails, set echo return value. + rundir_perm + "$obsdir"/bs_clouduploadworker --restart + rc_status + ;; + try-restart|reload) + $0 status + if test $? = 0; then + rundir_perm + "$obsdir"/bs_clouduploadworker --restart + else + rc_reset # Not running is not a failure. + fi + # Remember status and be quiet + rc_status + ;; + status) + echo -n "Checking for obsclouduploadworker and running processes: " + checkproc "$obsdir"/bs_clouduploadworker + rc_status -v + ;; + *) + echo "Usage: $0 {start|stop|status|try-restart|restart|reload}" + exit 1 + ;; +esac +rc_exit diff --git a/src/backend/bs_clouduploadserver b/src/backend/bs_clouduploadserver index 46d4f8ec0fa..45d0e5b824d 100755 --- a/src/backend/bs_clouduploadserver +++ b/src/backend/bs_clouduploadserver @@ -41,12 +41,6 @@ use BSHTTP; use strict; -$BSConfig::bsuser = $BSConfig::bsserviceuser; -$BSConfig::bsgroup = $BSConfig::bsservicegroup; - -BSUtil::set_fdatasync_before_rename() unless $BSConfig::disable_data_sync || $BSConfig::disable_data_sync; - - my $port = 5452; $port = $1 if $BSConfig::clouduploadserver =~ /:(\d+)$/; @@ -157,11 +151,19 @@ sub cloudupload_upload { } rename("$jobdir/.$jobid.file$$", "$jobdir/$jobid.file") || die("rename $jobdir/.$jobid.file$$ $jobdir/$jobid.file: $!\n"); $job->{'state'} = 'scheduled'; + delete $job->{'details'}; writexml("$jobdir/.$jobid$$", "$jobdir/$jobid", $job, $BSXML::clouduploadjob); close F; return $BSStdServer::return_ok; } +sub cloudupload_status { + my ($cgi, $jobid) = @_; + my ($job) = readxml("$jobdir/$jobid", $BSXML::clouduploadjob, 1) || readxml("$jobdir/done/$jobid", $BSXML::clouduploadjob, 1); + die("404 no such job\n") unless $job; + return ($job, $BSXML::clouduploadjob); +} + sub workerstatus { my ($cgi) = @_; my @daemonarchs = qw{clouduploadserver clouduploadworker}; @@ -212,10 +214,12 @@ my $dispatches = [ '!- GET:' => undef, '!- HEAD:' => undef, - '/serverstatus' => \&BSStdServer::serverstatus, - '/workerstatus arch*' => \&workerstatus, '!- POST:/cloudupload' => \&cloudupload_create, '!- PUT:/cloudupload/$job' => \&cloudupload_upload, + '/cloudupload/$job' => \&cloudupload_status, + + '/serverstatus' => \&BSStdServer::serverstatus, + '/workerstatus arch*' => \&workerstatus, ]; my $conf = { diff --git a/src/backend/bs_clouduploadworker b/src/backend/bs_clouduploadworker new file mode 100755 index 00000000000..c6db119b990 --- /dev/null +++ b/src/backend/bs_clouduploadworker @@ -0,0 +1,188 @@ +#!/usr/bin/perl + +BEGIN { + my ($wd) = $0 =~ m-(.*)/- ; + $wd ||= '.'; + unshift @INC, "$wd/build"; + unshift @INC, "$wd"; +} + +use XML::Structured ':bytes'; +use Data::Dumper; +use POSIX; +use Fcntl qw(:DEFAULT :flock); + +use BSConfiguration; +use BSUtil; +use BSStdRunner; +use BSRunner; +use BSXML; + +use strict; + +my $bsdir = $BSConfig::bsdir || "/srv/obs"; + +my $jobdir = "$BSConfig::bsdir/cloudupload"; +my $eventdir = "$BSConfig::bsdir/events"; +my $rundir = $BSConfig::rundir || "$BSConfig::bsdir/run"; + +my $maxchild = 4; +$maxchild = $BSConfig::clouduploadworker_maxchild if defined $BSConfig::clouduploadworker_maxchild; + +my $myeventdir = "$eventdir/clouduploadworker"; +my $jobdonedir = "$jobdir/cloudupload/done"; + +sub lsjobs { + return sort {$a <=> $b} grep {/^\d+$/} ls($jobdir); +} + +sub reap { + my ($chld, $chld_user) = @_; + my $pid; + while (($pid = waitpid(-1, POSIX::WNOHANG)) > 0) { + my $user = delete $chld->{$pid}; + delete $chld_user->{$user}->{$pid} if $chld_user->{$user}; + } +} + +sub startupload { + my ($conf, $job, $logfp) = @_; + my $jobid = $job->{'name'}; + my $pid; + if (!($pid = xfork())) { + close($logfp); + exec("upload", $job->{'user'}, $job->{'target'}, "$jobdir/$jobid.file", "$jobdir/$jobid.data", "$jobdir/$jobid.log"); + die("upload: $!\n"); + } + while (1) { + if (!waitpid($pid, 0)) { + next if $! == POSIX::EINTR; + last; + } + } + my $ex = $?; + local *JOBLOCK; + $job = BSUtil::lockopenxml(\*JOBLOCK, '<', "$jobdir/$jobid", $BSXML::clouduploadjob); + if ($job->{'state'} eq 'uploading') { + if ($ex) { + $job->{'state'} = 'failed'; + } else { + $job->{'state'} = 'succeeded'; + } + delete $job->{'pid'}; + } + mkdir_p($jobdonedir); + rename("$jobdir/$jobid.log", "$jobdonedir/$jobid.log"); + writexml("$jobdonedir/.$jobid$$", "$jobdonedir/$jobid", $job, $BSXML::clouduploadjob); + unlink("$jobdir/$jobid.log"); + unlink("$jobdir/$jobid.data"); + unlink("$jobdir/$jobid.file"); + unlink("$jobdir/$jobid"); + close JOBLOCK; + BSUtil::ping($conf->{'ping'}); + exit(0); +} + +sub checkjob { + my ($jobid, $job) = @_; + die("wrong job name $job->{'name'}\n") if $job->{'name'} ne $jobid; + die("no target\n") unless $job->{'target'}; + die("no user\n") unless $job->{'user'}; + die("no target data\n") unless -e "$jobdir/$jobid.data"; +} + +sub run { + my ($conf) = @_; + + my $ping = $conf->{'ping'}; + my $maxchild = $conf->{'maxchild'}; + my $maxchild_user = $conf->{'maxchild_user'}; + my %chld; + my %chld_user; + my $pid; + while(1) { + BSRunner::drainping($ping); + reap(\%chld, \%chld_user) if %chld; + + my @jobs = lsjobs(); + + my $now = time(); + my $havedelayed; + for my $jobid (@jobs) { + last if keys(%chld) >= $maxchild; + my $job = readxml("$jobdir/$jobid", $BSXML::clouduploadjob, 1); + next unless $job; + if ($job->{'due'} && $job->{'due'} > $now) { + $havedelayed = 1; + next; + } + if ($job->{'state'} ne 'scheduled' && $job->{'state'} ne 'waiting') { + next; + } + my $user = $job->{'user'}; + if ($maxchild_user) { + if (keys(%{$chld_user{$user} || {}}) >= $maxchild_user) { + $havedelayed = 1; + next; + } + } + + # create a new job + local *JOBLOCK; + $job = BSUtil::lockopenxml(\*JOBLOCK, '<', "$jobdir/$jobid", $BSXML::clouduploadjob, 1); + next unless $job; + if ($job->{'state'} ne 'scheduled' && $job->{'state'} ne 'waiting') { + close JOBBLOCK; + next; + } + eval { checkjob($jobid, $job) }; + if ($@) { + warn("bad job: $jobid $@\n"); + close JOBBLOCK; + next; + } + local *LOGLOCK; + unlink("$jobdir/.$jobid.log.$$"); + BSUtil::lockopen(\*LOGLOCK, '>', "$jobdir/.$jobid.log.$$"); + rename("$jobdir/.$jobid.log.$$", "$jobdir/$jobid.log") || die("rename $jobdir/.$jobid.log.$$ $jobdir/$jobid.log: $!\n"); + if (!($pid = xfork())) { + close JOBLOCK; + startupload($conf, $job, \*LOGLOCK); + exit(0); + } + close LOGLOCK; + $job->{'pid'} = $pid; + $job->{'state'} = 'uploading'; + writexml("$jobdir/.$jobid$$", "$jobdir/$jobid", $job, $BSXML::clouduploadjob); + close JOBLOCK; + $chld{$pid} = $user; + $chld_user{$user}->{$pid} = undef; + } + reap(\%chld, \%chld_user) if %chld; + + for my $fc (sort %{$conf->{'filechecks'} || {}}) { + next unless -e $fc; + $conf->{'filechecks'}->{$fc}->($conf, $fc); + } + + if ($havedelayed) { + BSRunner::waitping($ping, 10); + } else { + if ($conf->{'testmode'} && !%chld) { + print "test mode, all jobs processed, exiting...\n"; + last; + } + print "waiting for an event...\n"; + BSRunner::waitping($ping); + } + } +} + +my $conf = { + 'eventdir' => $myeventdir, + 'dispatches' => [], + 'maxchild' => $maxchild, + 'run' => \&run, +}; + +BSStdRunner::run('bs_clouduploadworker', \@ARGV, $conf);