diff --git a/bin/gbuild b/bin/gbuild index e3f5c16..84462ce 100755 --- a/bin/gbuild +++ b/bin/gbuild @@ -80,6 +80,14 @@ def build_one_configuration(suite, arch, build_desc, reference_datetime) system! "copy-to-target #{@quiet_flag} inputs/#{filename} build/" end + if Dir.exists?("cache/#{build_desc["name"]}") + system! "copy-to-target #{@quiet_flag} cache/#{build_desc["name"]}/ cache/" + end + + if Dir.exists?("cache/common") + system! "copy-to-target #{@quiet_flag} cache/common/ cache/" + end + info "Updating apt-get repository (log in var/install.log)" system! "on-target -u root apt-get update > var/install.log 2>&1" @@ -164,10 +172,12 @@ in_sums = [] build_dir = 'build' result_dir = 'result' +cache_dir = 'cache' FileUtils.rm_rf(build_dir) FileUtils.mkdir(build_dir) FileUtils.mkdir_p(result_dir) +FileUtils.mkdir_p(cache_dir) package_name = build_desc["name"] or raise "must supply name" package_name = sanitize(package_name, "package name") @@ -237,12 +247,18 @@ suites.each do |suite| info "Grabbing results" system! "copy-from-target #{@quiet_flag} out #{build_dir}" + info "Grabbing cache" + system! "copy-from-target #{@quiet_flag} cache/#{package_name}/ #{cache_dir}" + system! "copy-from-target #{@quiet_flag} cache/common/ #{cache_dir}" + base_manifest = File.read("var/base-#{suite}-#{arch}.manifest") base_manifests["#{suite}-#{arch}"] = base_manifest end end out_dir = File.join(build_dir, "out") +cache_common_dir = File.join(cache_dir, "common") +cache_package_dir = File.join(cache_dir, "#{package_name}") out_sums = {} info "Generating report" @@ -255,6 +271,24 @@ Dir.glob(File.join(out_dir, '**', '*'), File::FNM_DOTMATCH).sort.each do |file_i puts out_sums[file] unless @options[:quiet] end +Dir.glob(File.join(cache_common_dir, '**', '*'), File::FNM_DOTMATCH).sort.each do |file_in_out| + next if File.directory?(file_in_out) + file = file_in_out.sub(cache_common_dir + File::SEPARATOR, '') + file = sanitize_path(file, file_in_out) + out_sums[file] = `cd #{cache_common_dir} && sha256sum #{file}` + raise "failed to sum #{file}" unless $? == 0 + puts out_sums[file] unless @options[:quiet] +end + +Dir.glob(File.join(cache_package_dir, '**', '*'), File::FNM_DOTMATCH).sort.each do |file_in_out| + next if File.directory?(file_in_out) + file = file_in_out.sub(cache_package_dir + File::SEPARATOR, '') + file = sanitize_path(file, file_in_out) + out_sums[file] = `cd #{cache_package_dir} && sha256sum #{file}` + raise "failed to sum #{file}" unless $? == 0 + puts out_sums[file] unless @options[:quiet] +end + out_manifest = out_sums.keys.sort.map { |key| out_sums[key] }.join('') in_manifest = in_sums.join('') diff --git a/doc/CACHE b/doc/CACHE new file mode 100644 index 0000000..9c3cc42 --- /dev/null +++ b/doc/CACHE @@ -0,0 +1,23 @@ +Gitian includes two caches where build-assets may be saved for use with +subsequent builds. + +Common cache: +All descriptors share this cache. It can be useful for storing fetched sources, +sharing build assets between descriptors, etc. +To add or update files, copy them to ~/cache/common from the build script. + +Per-descriptor cache +Files installed to this cache can only be seen by this descriptor. Use this to +store assets created as part of the build process, to avoid having to rebuild +them in future builds. +To add or update files, copy them to ~/cache/$NAME from the build script, where +$NAME is the value of the descriptor's "name" key. + +Before each build, all files and folders in the cache directories will be +transferred to the VM. After each successful build, the caches will be +transferred back to the builder (overwriting existing files as necessary). + +It is entirely up to the user and the descriptors to verify that cached files +are actually suitable for re-use. Overwriting existing cached files is _highly_ +discouraged, as that would likely lead to non-deterministic results. Unique +filenames should always be used. diff --git a/target-bin/init-build.sh b/target-bin/init-build.sh index 142f2e8..bd27dc4 100644 --- a/target-bin/init-build.sh +++ b/target-bin/init-build.sh @@ -1,6 +1,7 @@ #!/bin/sh -rm -rf install out build +rm -rf install out build cache mkdir build mkdir out mkdir install +mkdir -p cache/common