Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
Replace error-prone file timestamp check by checksum when loading deps 
File modification timestamps are unreliable as file systems may have a
granularity of 2 seconds. In addition, packagers may limit time stamps to
certain ranges for making build results reproducible.

We've already used checksums instead of timestamps for precompilation units
themselves. Now we also use checksums for checking a precompilation unit's
source file.
  • Loading branch information
@niner
niner committed Apr 21, 2017
1 parent b2a64a1 commit ca0a743
Show file tree
Hide file tree
Showing 3 changed files with 60 additions and 20 deletions.
18 changes: 4 additions & 14 deletions src/core/CompUnit/PrecompilationRepository.pm
View file
Expand Up @@ -131,19 +131,8 @@ class CompUnit::PrecompilationRepository::Default does CompUnit::PrecompilationR
$RMD("Could not find $dependency.spec()") if $RMD and not $store;
return False unless $store;

if $resolve { # a repo changed, so maybe it's a change in our source file
my $modified = $file.modified;
$RMD("$file\nspec: $dependency.spec()\nmtime: $modified\nsince: $since")
if $RMD;

my $srcIO = CompUnit::RepositoryRegistry.file-for-spec($dependency.src) // $dependency.src.IO;
$RMD("source: $srcIO mtime: " ~ $srcIO.modified) if $RMD and $srcIO.e;
return False if not $srcIO.e or $modified < $srcIO.modified;
}

my $dependency-precomp = $store.load-unit($compiler-id, $dependency.id);
$RMD("dependency checksum $dependency.checksum() unit: $dependency-precomp.checksum()") if $RMD;
return False if $dependency-precomp.checksum ne $dependency.checksum;
return False unless $dependency-precomp.is-up-to-date($dependency, :check-source($resolve));

@dependencies.push: $dependency-precomp;
}
Expand Down Expand Up @@ -201,7 +190,7 @@ class CompUnit::PrecompilationRepository::Default does CompUnit::PrecompilationR
}
else {
if $*RAKUDO_MODULE_DEBUG -> $RMD {
$RMD("Outdated precompiled $unit\nmtime: $modified\nsince: $since")
$RMD("Outdated precompiled $unit\nmtime: {$modified}{$since ?? "\nsince: $since" !! ''}")
}
$unit.close;
fail "Outdated precompiled $unit";
Expand Down Expand Up @@ -236,6 +225,7 @@ class CompUnit::PrecompilationRepository::Default does CompUnit::PrecompilationR
self.store.unlock;
return True;
}
my $source-checksum = nqp::sha1($path.slurp(:enc<iso-8859-1>));
my $bc = "$io.bc".IO;

$lle //= Rakudo::Internals.LL-EXCEPTION;
Expand Down Expand Up @@ -307,7 +297,7 @@ class CompUnit::PrecompilationRepository::Default does CompUnit::PrecompilationR
self.store.store-unit(
$compiler-id,
$id,
self.store.new-unit(:$id, :@dependencies, :bytecode($bc.slurp(:bin))),
self.store.new-unit(:$id, :@dependencies, :$source-checksum, :bytecode($bc.slurp(:bin))),
);
$bc.unlink;
self.store.store-repo-id($compiler-id, $id, :repo-id($*REPO.id));
Expand Down
39 changes: 33 additions & 6 deletions src/core/CompUnit/PrecompilationStore/File.pm
View file
Expand Up @@ -6,10 +6,20 @@ class CompUnit::PrecompilationStore::File does CompUnit::PrecompilationStore {
has CompUnit::PrecompilationDependency @!dependencies;
has $!initialized = False;
has $.checksum;
has $.source-checksum;
has $!bytecode;
has $!store;
has Lock $!update-lock = Lock.new;

submethod BUILD(CompUnit::PrecompilationId :$!id, IO::Path :$!path, :@!dependencies, :$!bytecode --> Nil) {
submethod BUILD(
CompUnit::PrecompilationId :$!id,
IO::Path :$!path,
:$!source-checksum,
:@!dependencies,
:$!bytecode,
:$!store,
--> Nil
) {
if $!bytecode {
$!initialized = True;
$!checksum = nqp::sha1($!bytecode.decode("latin-1"));
Expand All @@ -29,8 +39,9 @@ class CompUnit::PrecompilationStore::File does CompUnit::PrecompilationStore {
return if $!initialized;
self!open(:r) unless $!file;

$!checksum = $!file.get;
my $dependency = $!file.get;
$!checksum = $!file.get;
$!source-checksum = $!file.get;
my $dependency = $!file.get;
while $dependency {
@!dependencies.push: CompUnit::PrecompilationDependency::File.deserialize($dependency);
$dependency = $!file.get;
Expand All @@ -56,6 +67,11 @@ class CompUnit::PrecompilationStore::File does CompUnit::PrecompilationStore {
$!file
}

method source-checksum() is rw {
self!read-dependencies;
$!source-checksum
}

method checksum() is rw {
self!read-dependencies;
$!checksum
Expand All @@ -75,12 +91,19 @@ class CompUnit::PrecompilationStore::File does CompUnit::PrecompilationStore {
method save-to(IO::Path $precomp-file) {
my $handle = $precomp-file.open(:w);
$handle.print($!checksum ~ "\n");
$handle.print($!source-checksum ~ "\n");
$handle.print($_.serialize ~ "\n") for @!dependencies;
$handle.print("\n");
$handle.write($!bytecode);
$handle.close;
$!path = $precomp-file;
}

method is-up-to-date(CompUnit::PrecompilationDependency $dependency, Bool :$check-source --> Bool) {
my $result = self.CompUnit::PrecompilationUnit::is-up-to-date($dependency, :$check-source);
$!store.remove-from-cache($.id) unless $result;
$result
}
}

has IO::Path $.prefix is required;
Expand All @@ -95,7 +118,7 @@ class CompUnit::PrecompilationStore::File does CompUnit::PrecompilationStore {
}

method new-unit(|c) {
CompUnit::PrecompilationUnit::File.new(|c)
CompUnit::PrecompilationUnit::File.new(|c, :store(self))
}

method !dir(CompUnit::PrecompilationId $compiler-id,
Expand Down Expand Up @@ -140,7 +163,7 @@ class CompUnit::PrecompilationStore::File does CompUnit::PrecompilationStore {
%!loaded{$precomp-id} //= do {
my $path = self.path($compiler-id, $precomp-id);
$path ~~ :e
?? CompUnit::PrecompilationUnit::File.new(:id($precomp-id), :$path)
?? CompUnit::PrecompilationUnit::File.new(:id($precomp-id), :$path, :store(self))
!! Nil
}
}
Expand All @@ -158,6 +181,10 @@ class CompUnit::PrecompilationStore::File does CompUnit::PrecompilationStore {
}
}

method remove-from-cache(CompUnit::PrecompilationId $precomp-id) {
$!update-lock.protect: { %!loaded{$precomp-id}:delete };
}

method destination(CompUnit::PrecompilationId $compiler-id,
CompUnit::PrecompilationId $precomp-id,
Str :$extension = ''
Expand Down Expand Up @@ -198,7 +225,7 @@ class CompUnit::PrecompilationStore::File does CompUnit::PrecompilationStore {
my $precomp-file = self!file($compiler-id, $precomp-id, :extension<.tmp>);
$unit.save-to($precomp-file);
$precomp-file.rename(self!file($compiler-id, $precomp-id));
$!update-lock.protect: { %!loaded{$precomp-id}:delete };
self.remove-from-cache($precomp-id);
}

method store-repo-id(CompUnit::PrecompilationId $compiler-id,
Expand Down
23 changes: 23 additions & 0 deletions src/core/CompUnit/PrecompilationUnit.pm
View file
Expand Up @@ -21,6 +21,7 @@ role CompUnit::PrecompilationDependency {
method id(--> CompUnit::PrecompilationId:D) { ... }
method src(--> Str:D) { ... }
method spec(--> CompUnit::DependencySpecification:D) { ... }
method checksum(--> Str:D) { ... }
method Str() {
"$.id $.src $.spec"
}
Expand All @@ -35,8 +36,30 @@ role CompUnit::PrecompilationUnit {
method dependencies(--> Array[CompUnit::PrecompilationDependency]) { ... }
method bytecode(--> Buf:D) { ... }
method checksum(--> Str:D) { ... }
method source-checksum(--> Str:D) { ... }
method bytecode-handle(--> IO::Handle:D) { ... }
method close(--> Nil) { ... }
method is-up-to-date(CompUnit::PrecompilationDependency $dependency, Bool :$check-source --> Bool) {
my $RMD = $*RAKUDO_MODULE_DEBUG;
if $check-source { # a repo changed, so maybe it's a change in our source file
my $source-checksum = $.source-checksum;

my $srcIO = CompUnit::RepositoryRegistry.file-for-spec($dependency.src) // $dependency.src.IO;
unless $srcIO {
return False unless $srcIO.e;
}
my $current-source-checksum := nqp::sha1($srcIO.slurp(:enc<iso-8859-1>));
$RMD(
"$.path\nspec: $dependency.spec()\nsource: $srcIO\n"
~ "source-checksum: $source-checksum\ncurrent-source-checksum: $current-source-checksum"
) if $RMD;
return False if $source-checksum ne $current-source-checksum;
}

$RMD("dependency checksum $dependency.checksum() unit: $.checksum()") if $RMD;

$.checksum eq $dependency.checksum
}
}

class CompUnit::PrecompilationDependency::File does CompUnit::PrecompilationDependency {
Expand Down

0 comments on commit ca0a743

Please sign in to comment.