Skip to content

Commit

Permalink
expire improvements [5]: keep longest copy of output.
Browse files Browse the repository at this point in the history
when collecting %seen files =~ /^yyyymmdd-/, notice the longest
 (but don't do anything to either copy yet);
when cleaning duplicate .tmp.file,
 unlink it if it is shorter (an old copy from fetch/expire race)
 but replace the done_file if it is longer (patch over our rename to .staletmp)
  • Loading branch information
mcast committed May 25, 2011
1 parent c8ed4dc commit def23e1
Showing 1 changed file with 37 additions and 13 deletions.
50 changes: 37 additions & 13 deletions expire
Expand Up @@ -54,9 +54,22 @@ foreach my $dir (glob("*")) {

foreach my $file (glob("$dir/*")) {
my $n = -M $file;
$seen{$1} = $file if $file =~ m{/(\d+-\d+\.\d+)($|\.)};
$newest = $n if ($newest > $n);
$newest_stamp = $n if ($newest_stamp > $n);

if ($file =~ m{/(\d+-\d+\.\d+)($|\.)}) {
# Remember the date-time.rnd part
my $dt = $1;
if (defined $seen{$dt}) {
# Erk, two of them. Remember the bigger one.
#
# Probably renamed by expire from .tmp.file to
# file.staletmp
$seen{$dt} = $file if -s $file > -s $seen{$dt};
} else {
$seen{$dt} = $file;
}
}
}

my @WARN;
Expand All @@ -66,18 +79,29 @@ foreach my $dir (glob("*")) {
next unless $dt;
my $begin = stamp2u($dt);

if ($seen{"$dt.$rnd"}) {
# This .tmp.file has been renamed elsewhere, but we have
# both copies because we fetched the .tmp.file while the
# job was running.
#
# Delete the .tmp.file because it's stale.
unlink($file) or push @WARN, "Cannot unlink($file): $!";

warn "Race file cleaned: $file\n";
# this makes expire noisy, which it previously was not;
# mine runs under 'add' and immediately after data-from
# so that may not matter..?
if (my $done_file = $seen{"$dt.$rnd"}) {
# The rename of .tmp.file has been done elsewhere, but we
# have both copies because we fetched the .tmp.file while
# the job was running.
if (-s $file > -s $done_file) {
# .tmp.file is longer, repeat the rename. This is
# probably a .staletmp file.
if (rename($file, $done_file)) {
push @WARN, "Repeated rename $file -> $done_file\n"
} else {
push @WARN, "Cannot repeat rename $file -> $done_file: $!\n";
}
utime $^T, $^T, $done_file; # bump to the top of the RSS

} else {
# Delete the .tmp.file because it's stale.
unlink($file) or push @WARN, "Cannot unlink($file): $!\n";

warn "Race file cleaned: $file\n";
# this makes expire noisy, which it previously was
# not; mine runs under 'add' and immediately after
# data-from so that may not matter..?
}

} elsif ($^T - $begin > 0.3 * 86400) {
# Guess what happened to its job?
Expand Down

0 comments on commit def23e1

Please sign in to comment.