Browse files

Added more of David's 'Chunks & Job Priority' patch.

  • Loading branch information...
1 parent b90fadb commit 4310ddd8a6ebfcf2c5acc30f64ab87483da452a1 Nick committed Apr 15, 2011
Showing with 41 additions and 7 deletions.
  1. +17 −2 bin/bric_queued
  2. +11 −2 bin/bric_republish
  3. +13 −3 bin/bric_soap
View
19 bin/bric_queued
@@ -50,6 +50,13 @@ Specifies a delay in seconds after finding the queue empty.
Run a single job of C<type> where C<type> is one of 'pub' or 'dist'.
Implies -verbose.
+=head2 -c <number> | --chunks <number>
+
+Specify the maximum number of jobs to select in a single run. This is most
+useful for reducing the memory overhead when a lot of jobs are scheduled, as
+well as allowing for higher-priority jobs not to get caught behind a huge
+pileup of low-priority jobs. Does not apply when C<--single> is specified.
+
=head2 -v | --verbose
Turn on verbose mode for debugging.
@@ -163,6 +170,7 @@ my $Verbose = undef;
my $Logfile = '/dev/null';
my $HelpMode = undef;
my $CaughtSignal = 0; # to be set by the signal handler
+my $Chunks;
##############################################################################
# The script.
@@ -173,6 +181,7 @@ GetOptions(
"pid|p=s" => \$Pidfile,
"delay|d=i" => \$Delay,
"single|s=s" => \$SingleJobMode,
+ 'chunks|c=i' => \$Chunks,
"verbose|v" => \$Verbose,
"log|l=s" => \$Logfile,
"help|h" => \$HelpMode,
@@ -220,15 +229,16 @@ sub run_as_daemon {
my $pkg = fork_to_dist();
login();
while (1) {
- # XXX Add a Limit parameter here? Seems like we should be able to
- # force it to do no more than a few at a time.
+ my $has_jobs = 0;
for my $job ($pkg->list({
sched_time => [undef, strfdate()],
comp_time => undef,
failed => '0',
executing => '0',
+ Limit => $Chunks,
})) {
logit('Executing ' . $job->get_name . "\n") if $Verbose;
+ $has_jobs++;
eval {
$job->execute_me;
commit_events();
@@ -237,6 +247,9 @@ sub run_as_daemon {
terminate() if $CaughtSignal;
}
+ # If we found jobs, assume there are more. Don't wait to execute them.
+ next if $has_jobs && (!$Chunks || $Chunks == $has_jobs);
+
# Flush out the publish_another queue.
if ($pkg eq PUB_PKG) {
eval {
@@ -302,6 +315,7 @@ sub run_single_job {
comp_time => undef,
failed => '0',
executing => '0',
+ Limit => 1,
});
exit unless $job;
logit('Executing ' . $job->get_name . "\n") if $Verbose;
@@ -498,3 +512,4 @@ __END__
=head1 Author
Mark Jaroski <jaroskim@who.int>
+
View
13 bin/bric_republish
@@ -42,6 +42,11 @@ bric_republish [options]
--published-only - Deprecated; kept for backwards compatibility; Only
the published version of all stories will be published.
+ --priority - Priority for publish. Must be between 1 and 5. Defaults to
+ the priorities of the assets published. Useful for bulk
+ publishing with a lower priority, so that normal and
+ high-priority jobs published by users take priority.
+
--chunks - publish stories in chunks of this many. Defaults to 0,
which means to process them all at once. This option can
be used to avoid timing out on large jobs.
@@ -111,7 +116,7 @@ BEGIN {
our $no_media = 0;
our $timeout = 30;
our $chunks = 0;
- our ($element, $story_id, $category, $help, $man, $site);
+ our ($element, $story_id, $category, $help, $man, $site, $priority);
GetOptions("help" => \$help,
"man" => \$man,
"verbose+" => \$VERBOSE,
@@ -123,6 +128,7 @@ BEGIN {
"no-media" => \$no_media,
"story-id=s" => \$story_id,
"published-only" => \my $published_only,
+ 'priority=i' => \$priority,
"timeout=s" => \$timeout,
"chunks=s" => \$chunks,
"site=s" => \$site,
@@ -245,7 +251,7 @@ sub get_media_ids {
# publish stories and media found
sub publish_assets {
- our ($soap, @story_ids, @media_ids, $chunks);
+ our ($soap, @story_ids, @media_ids, $chunks, $priority);
# do nothing if we've got nothing
return unless @story_ids;
@@ -272,6 +278,7 @@ sub publish_assets {
my $r = $soap->publish(
name( publish_ids => [ @ids[$start .. $end] ] ),
name( published_only => 1 ), # Always published_version only.
+ ($priority ? name(priority => $priority) : ()),
);
_print_fault($r) if $r->fault;
}
@@ -285,6 +292,7 @@ sub publish_assets {
my $r = $soap->publish(
name( publish_ids => \@ids ),
name( published_only => 1 ), # Always published_version only.
+ ($priority ? name(priority => $priority) : ()),
);
_print_fault($r) if $r->fault;
}
@@ -331,3 +339,4 @@ sub _print_fault {
$r->faultstring;
}
}
+
View
16 bin/bric_soap
@@ -97,6 +97,12 @@ Options:
--to-preview - use to_preview option for workflow publish
+ --priority - Priority for workflow publish. Must be between 1 and 5.
+ Defaults to the priorities of the assets published.
+ Useful for bulk publishing with a lower priority, so
+ that normal and high-priority jobs published by users
+ take priority.
+
--publish-date - date and time to publish assets for workflow
publish. Use format CCYY-MM-DDThh:mm:ssZ, where
the "Z" stands for UTC (GMT).
@@ -247,6 +253,7 @@ BEGIN {
our $without_file = 0;
our $use_related_uri = 0;
our $to_preview = 0;
+ our $priority;
our $publish_date;
our $published_only = 0;
our $timeout = 30;
@@ -274,6 +281,7 @@ BEGIN {
"use-related-uri" => \$use_related_uri,
"all" => sub { $with_related_stories = $with_related_media = 1; },
"to-preview" => \$to_preview,
+ "priority=i" => \$priority,
"publish-date=s" => \$publish_date,
"published-only" => \$published_only,
"desk=s" => \$desk,
@@ -298,9 +306,9 @@ BEGIN {
}
our (
$module, $command, %search, $with_related_stories, $with_related_media,
- $without_file, $to_preview, $publish_date, $published_only, $username,
- $password, $server, $verbose, $desk, $workflow, $timeout, $use_cookie,
- $save_cookie, $chunks, $continue, $use_related_uri
+ $without_file, $to_preview, $priority, $publish_date, $published_only,
+ $username, $password, $server, $verbose, $desk, $workflow, $timeout,
+ $use_cookie, $save_cookie, $chunks, $continue, $use_related_uri
);
use SOAP::Lite ($verbose > 2 ? (trace => [qw(debug)]) : ()),
@@ -569,6 +577,7 @@ sub publish {
push @opts, name(publish_related_stories => 1) if $with_related_stories;
push @opts, name(publish_related_media => 1) if $with_related_media;
push @opts, name(to_preview => 1) if $to_preview;
+ push @opts, name(priority => $priority ) if $priority;
push @opts, name(publish_date => $publish_date) if $publish_date;
push @opts, name(published_only => 1) if $published_only;
@@ -716,3 +725,4 @@ sub _handle_fault {
# return an empty object to keep things humming along
return SOAP::SOM->new();
}
+

0 comments on commit 4310ddd

Please sign in to comment.