Permalink
Cannot retrieve contributors at this time
executable file
432 lines (355 sloc)
14.1 KB
Name already in use
A tag already exists with the provided branch name. Many Git commands accept both tag and branch names, so creating this branch may cause unexpected behavior. Are you sure you want to create this branch?
gitian-builder/bin/gbuild
Go to fileThis commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env ruby | |
require 'optparse' | |
require 'yaml' | |
require 'fileutils' | |
require 'pathname' | |
@options = {:num_procs => 2, :memory => 2000} | |
@bitness = { | |
'i386' => 32, | |
'amd64' => 64, | |
'linux64' => 64, | |
} | |
@arches = { | |
'i386' => 'i386', | |
'amd64' => 'x86_64', | |
'linux64' => 'linux64', | |
} | |
def system!(cmd) | |
system(cmd) or raise "failed to run #{cmd}" | |
end | |
def sanitize(str, where) | |
raise "unsanitary string in #{where}" if (str =~ /[^\w.-]/) | |
str | |
end | |
def sanitize_path(str, where) | |
raise "unsanitary string in #{where}" if (str =~ /[^@\w\/.:+-]/) | |
str | |
end | |
def info(str) | |
puts str unless @options[:quiet] | |
end | |
def build_one_configuration(suite, arch, build_desc) | |
FileUtils.rm_f("var/install.log") | |
FileUtils.rm_f("var/build.log") | |
bits = @bitness[arch] or raise "unknown architecture ${arch}" | |
if ENV["USE_LXC"] == "1" | |
ENV["LXC_ARCH"] = arch | |
ENV["LXC_SUITE"] = suite | |
end | |
suitearch = "#{suite}-#{arch}" | |
info "Stopping target if it is up" | |
system "stop-target" | |
sleep 1 | |
unless @options[:skip_image] | |
info "Making a new image copy" | |
system! "make-clean-vm --suite #{suite} --arch #{arch}" | |
end | |
info "Starting target" | |
system! "start-target #{bits} #{suitearch}&" | |
$stdout.write "Checking if target is up" | |
(1..30).each do | |
system "on-target true 2> /dev/null" and break | |
sleep 2 | |
$stdout.write '.' | |
end | |
info '' | |
system! "on-target true" | |
system! "on-target -u root tee -a /etc/sudoers.d/#{ENV['DISTRO'] || 'ubuntu'} > /dev/null << EOF | |
%#{ENV['DISTRO'] || 'ubuntu'} ALL=(ALL) NOPASSWD: ALL | |
EOF" if build_desc["sudo"] and @options[:allow_sudo] | |
info "Preparing build environment" | |
system! "on-target setarch #{@arches[arch]} bash < target-bin/init-build.sh" | |
build_desc["files"].each do |filename| | |
filename = sanitize(filename, "files section") | |
system! "copy-to-target #{@quiet_flag} inputs/#{filename} build/" | |
end | |
if build_desc["enable_cache"] | |
if File.directory?("cache/#{build_desc["name"]}") | |
system! "copy-to-target #{@quiet_flag} cache/#{build_desc["name"]}/ cache/" | |
end | |
if File.directory?("cache/common") | |
system! "copy-to-target #{@quiet_flag} cache/common/ cache/" | |
end | |
end | |
if build_desc["multiarch"] | |
info "Adding multiarch support (log in var/install.log)" | |
for a in build_desc["multiarch"] | |
system! "on-target -u root dpkg --add-architecture #{a} >> var/install.log 2>&1" | |
end | |
end | |
if build_desc["repositories"] | |
info "Adding repositories to the sources list (log in var/install.log)" | |
for r in build_desc["repositories"] | |
system! "on-target -u root tee -a /etc/apt/sources.list >> var/install.log 2>&1 << EOF | |
#{r["source"]} | |
EOF" | |
end | |
end | |
info "Updating apt-get repository (log in var/install.log)" | |
system! "on-target -u root apt-get update >> var/install.log 2>&1" | |
info "Installing additional packages (log in var/install.log)" | |
system! "on-target -u root -e DEBIAN_FRONTEND=noninteractive apt-get --no-install-recommends -y install #{build_desc["packages"].join(" ")} >> var/install.log 2>&1" | |
if build_desc["repositories"] | |
for r in build_desc["repositories"] | |
info "Installing additional packages from repository #{r["distribution"]} (log in var/install.log)" | |
system! "on-target -u root -e DEBIAN_FRONTEND=noninteractive apt-get -t #{r["distribution"]} --no-install-recommends -y install #{r["packages"].join(" ")} >> var/install.log 2>&1" | |
end | |
end | |
if build_desc["alternatives"] | |
info "Set alternatives (log in var/install.log)" | |
for a in build_desc["alternatives"] | |
system! "on-target -u root update-alternatives --set #{a["package"]} #{a["path"]} >> var/install.log 2>&1" | |
end | |
end | |
if @options[:upgrade] || system("on-target -u root '[ ! -e /var/cache/gitian/initial-upgrade ]'") | |
info "Upgrading system, may take a while (log in var/install.log)" | |
system! "on-target -u root bash < target-bin/upgrade-system.sh >> var/install.log 2>&1" | |
end | |
info "Creating package manifest" | |
system! "on-target -u root bash < target-bin/grab-packages.sh > var/base-#{suitearch}.manifest" | |
info "Creating build script (var/build-script)" | |
File.open("var/build-script", "w") do |script| | |
script.puts "#!/bin/bash" | |
script.puts "set -e" | |
script.puts "export LANG='en_US.UTF-8'" | |
script.puts "export LC_ALL='en_US.UTF-8'" | |
script.puts "umask 002" | |
script.puts "export OUTDIR=$HOME/out" | |
script.puts "GBUILD_BITS=#{bits}" | |
if build_desc["enable_cache"] | |
script.puts "GBUILD_CACHE_ENABLED=1" | |
script.puts "GBUILD_PACKAGE_CACHE=$HOME/cache/#{build_desc["name"]}" | |
script.puts "GBUILD_COMMON_CACHE=$HOME/cache/common" | |
end | |
script.puts "MAKEOPTS=(-j#{@options[:num_procs]})" | |
script.puts "NUM_PROCS=#{@options[:num_procs]}" | |
script.puts "NUM_MEM=#{@options[:memory]}" | |
script.puts | |
author_date = nil | |
build_desc["remotes"].each do |remote| | |
dir = sanitize(remote["dir"], remote["dir"]) | |
author_date = `cd inputs/#{dir} && TZ=UTC git log --date='format-local:%F %T' --format="%ad" -1`.strip | |
raise "error looking up author date in #{dir}" unless $?.exitstatus == 0 | |
system! "copy-to-target #{@quiet_flag} inputs/#{dir} build/" | |
script.puts "(cd build/#{dir} && git reset -q --hard && git clean -q -f -d)" | |
end | |
script.puts | |
ref_datetime = build_desc["reference_datetime"] || author_date | |
(ref_date, ref_time) = ref_datetime.split | |
script.puts "REFERENCE_DATETIME='#{ref_datetime}'" | |
script.puts "REFERENCE_DATE='#{ref_date}'" | |
script.puts "REFERENCE_TIME='#{ref_time}'" | |
script.puts | |
script.puts "cd build" | |
script.puts build_desc["script"] | |
end | |
info "Running build script (log in var/build.log)" | |
if ENV["CI"] | |
system! "on-target setarch #{@arches[arch]} bash -x < var/build-script 2>&1 | tee var/build.log" | |
else | |
system! "on-target setarch #{@arches[arch]} bash -x < var/build-script > var/build.log 2>&1" | |
end | |
end | |
################################ | |
OptionParser.new do |opts| | |
opts.banner = "Usage: build [options] <build-description>.yml" | |
opts.on("--allow-sudo", "override SECURITY on the target VM and allow the use of sudo with no password for the default user") do |v| | |
@options[:allow_sudo] = v | |
end | |
opts.on("-i", "--skip-image", "reuse current target image") do |v| | |
@options[:skip_image] = v | |
end | |
opts.on("--upgrade", "upgrade guest with latest packages") do |v| | |
@options[:upgrade] = v | |
end | |
opts.on("-q", "--quiet", "be quiet") do |v| | |
@options[:quiet] = v | |
end | |
opts.on("-j PROCS", "--num-make PROCS", "number of processes to use") do |v| | |
@options[:num_procs] = v | |
end | |
opts.on("-m MEM", "--memory MEM", "memory to allocate in MiB") do |v| | |
@options[:memory] = v | |
end | |
opts.on("-c PAIRS", "--commit PAIRS", "comma separated list of DIRECTORY=COMMIT pairs") do |v| | |
@options[:commit] = v | |
end | |
opts.on("-u PAIRS", "--url PAIRS", "comma separated list of DIRECTORY=URL pairs") do |v| | |
@options[:url] = v | |
end | |
opts.on("-o", "--cache-read-only", "only use existing cache files, do not update them") do |v| | |
@options[:cache_ro] = v | |
end | |
opts.on("--skip-fetch", "skip fetching the latest git objects and refs from the remote source") do |v| | |
@options[:skip_fetch] = v | |
end | |
opts.on("--fetch-branches", "fetch branches from the remote source") do |v| | |
@options[:fetch_branches] = v | |
end | |
opts.on("--fetch-tags", "fetch tags from the remote source") do |v| | |
@options[:fetch_tags] = v | |
end | |
opts.on("--skip-cleanup", "skip cleaning up the target VM. this may be useful for copying additional files from the target after the build") do |v| | |
@options[:skip_cleanup] = v | |
end | |
end.parse! | |
if ENV["USE_LXC"] != "1" and ENV["USE_DOCKER"] != "1" and ENV["USE_VBOX"] != "1" and !File.exist?("/dev/kvm") | |
$stderr.puts "\n************* WARNING: kvm not loaded, this will probably not work out\n\n" | |
end | |
base_dir = Pathname.new(__FILE__).expand_path.dirname.parent | |
libexec_dir = base_dir + 'libexec' | |
ENV['PATH'] = libexec_dir.to_s + ":" + ENV['PATH'] | |
ENV['GITIAN_BASE'] = base_dir.to_s | |
ENV['NPROCS'] = @options[:num_procs].to_s | |
ENV['VMEM'] = @options[:memory].to_s | |
@quiet_flag = @options[:quiet] ? "-q" : "" | |
build_desc_file = ARGV.shift or raise "must supply YAML build description file" | |
build_desc = YAML.load_file(build_desc_file) | |
in_sums = [] | |
build_dir = 'build' | |
result_dir = 'result' | |
cache_dir = 'cache' | |
enable_cache = build_desc["enable_cache"] | |
FileUtils.rm_rf(build_dir) | |
FileUtils.mkdir(build_dir) | |
FileUtils.mkdir_p(result_dir) | |
package_name = build_desc["name"] or raise "must supply name" | |
package_name = sanitize(package_name, "package name") | |
if enable_cache | |
FileUtils.mkdir_p(File.join(cache_dir, "common")) | |
FileUtils.mkdir_p(File.join(cache_dir, package_name)) | |
end | |
distro = build_desc["distro"] || "ubuntu" | |
suites = build_desc["suites"] or raise "must supply suites" | |
archs = build_desc["architectures"] or raise "must supply architectures" | |
build_desc["reference_datetime"] or build_desc["remotes"].size > 0 or raise "must supply `reference_datetime` or `remotes`" | |
docker_image_digests = build_desc["docker_image_digests"] || [] | |
# if docker_image_digests are supplied, it must be the same length as suites | |
if docker_image_digests.size > 0 and suites.size != docker_image_digests.size | |
raise "`suites` and `docker_image_digests` must both be the same size if both are supplied" | |
elsif ENV["USE_DOCKER"] == "1" and docker_image_digests.size > 0 and suites.size == docker_image_digests.size | |
suites = docker_image_digests | |
end | |
ENV['DISTRO'] = distro | |
desc_sum = `sha256sum #{build_desc_file}` | |
desc_sum = desc_sum.sub(build_desc_file, "#{package_name}-desc.yml") | |
in_sums << desc_sum | |
build_desc["files"].each do |filename| | |
filename = sanitize(filename, "files section") | |
in_sums << `cd inputs && sha256sum #{filename}` | |
end | |
commits = {} | |
if @options[:commit] | |
@options[:commit].split(',').each do |pair| | |
(dir, commit) = pair.split('=') | |
commits[dir] = commit | |
end | |
end | |
urls = {} | |
if @options[:url] | |
@options[:url].split(',').each do |pair| | |
(dir, url) = pair.split('=') | |
urls[dir] = url | |
end | |
end | |
build_desc["remotes"].each do |remote| | |
if !remote["commit"] | |
remote["commit"] = commits[remote["dir"]] | |
raise "must specify a commit for directory #{remote["dir"]}" unless remote["commit"] | |
end | |
if urls[remote["dir"]] | |
remote["url"] = urls[remote["dir"]] | |
end | |
dir = sanitize(remote["dir"], remote["dir"]) | |
commit = sanitize(remote["commit"], remote["commit"]) | |
unless File.exist?("inputs/#{dir}") | |
system!("git init inputs/#{dir}") | |
end | |
if !@options[:skip_fetch] | |
if @options[:fetch_branches] | |
system!("cd inputs/#{dir} && git fetch -f --update-head-ok #{sanitize_path(remote["url"], remote["url"])} +refs/heads/*:refs/heads/*") | |
end | |
if @options[:fetch_tags] | |
system!("cd inputs/#{dir} && git fetch -f --update-head-ok #{sanitize_path(remote["url"], remote["url"])} +refs/tags/*:refs/tags/*") | |
end | |
system!("cd inputs/#{dir} && git fetch -f --update-head-ok #{sanitize_path(remote["url"], remote["url"])} #{commit}") | |
system!("cd inputs/#{dir} && git checkout -q FETCH_HEAD") | |
else | |
system!("cd inputs/#{dir} && git checkout -q #{commit}") | |
end | |
system!("cd inputs/#{dir} && git submodule update --init --recursive --force") | |
commit = `cd inputs/#{dir} && git log --format=%H -1`.strip | |
in_sums << "git:#{commit} #{dir}" | |
end | |
base_manifests = YAML::Omap.new | |
suites.each do |suite| | |
suite = sanitize(suite, "suite") | |
archs.each do |arch| | |
info "--- Building for #{suite} #{arch} ---" | |
arch = sanitize(arch, "architecture") | |
# Build! | |
build_one_configuration(suite, arch, build_desc) | |
info "Grabbing results from target" | |
system! "copy-from-target #{@quiet_flag} out #{build_dir}" | |
if enable_cache && !@options[:cache_ro] | |
info "Grabbing cache from target" | |
system! "copy-from-target #{@quiet_flag} cache/#{package_name}/ #{cache_dir}" | |
system! "copy-from-target #{@quiet_flag} cache/common/ #{cache_dir}" | |
end | |
base_manifest = File.read("var/base-#{suite}-#{arch}.manifest") | |
base_manifests["#{suite}-#{arch}"] = base_manifest | |
end | |
end | |
unless @options[:skip_cleanup] | |
info "Cleaning up target" | |
system "stop-target" | |
end | |
out_dir = File.join(build_dir, "out") | |
out_sums = {} | |
cache_common_dir = File.join(cache_dir, "common") | |
cache_package_dir = File.join(cache_dir, "#{package_name}") | |
cache_common_sums = {} | |
cache_package_sums = {} | |
info "Generating report" | |
Dir.glob(File.join(out_dir, '**', '*'), File::FNM_DOTMATCH).sort.each do |file_in_out| | |
next if File.directory?(file_in_out) | |
file = file_in_out.sub(out_dir + File::SEPARATOR, '') | |
file = sanitize_path(file, file_in_out) | |
out_sums[file] = `cd #{out_dir} && sha256sum #{file}` | |
raise "failed to sum #{file}" unless $? == 0 | |
puts out_sums[file] unless @options[:quiet] | |
end | |
if enable_cache | |
Dir.glob(File.join(cache_common_dir, '**', '*'), File::FNM_DOTMATCH).sort.each do |file_in_out| | |
next if File.directory?(file_in_out) | |
file = file_in_out.sub(cache_common_dir + File::SEPARATOR, '') | |
file = sanitize_path(file, file_in_out) | |
cache_common_sums[file] = `cd #{cache_common_dir} && sha256sum #{file}` | |
raise "failed to sum #{file}" unless $? == 0 | |
end | |
Dir.glob(File.join(cache_package_dir, '**', '*'), File::FNM_DOTMATCH).sort.each do |file_in_out| | |
next if File.directory?(file_in_out) | |
file = file_in_out.sub(cache_package_dir + File::SEPARATOR, '') | |
file = sanitize_path(file, file_in_out) | |
cache_package_sums[file] = `cd #{cache_package_dir} && sha256sum #{file}` | |
raise "failed to sum #{file}" unless $? == 0 | |
end | |
end | |
out_manifest = out_sums.keys.sort.map { |key| out_sums[key] }.join('') | |
in_manifest = in_sums.join('') | |
cache_common_manifest = cache_common_sums.keys.sort.map { |key| cache_common_sums[key] }.join('') | |
cache_package_manifest = cache_package_sums.keys.sort.map { |key| cache_package_sums[key] }.join('') | |
# Use Omap to keep result deterministic | |
report = YAML::Omap[ | |
'out_manifest', out_manifest, | |
'in_manifest', in_manifest, | |
'base_manifests', base_manifests, | |
'cache_common_manifest', cache_common_manifest, | |
'cache_package_manifest', cache_package_manifest, | |
] | |
result_file = "#{package_name}-res.yml" | |
File.open(File.join(result_dir, result_file), "w") do |io| | |
io.write report.to_yaml | |
end | |
system!("cd #{result_dir} && sha256sum #{result_file}") unless @options[:quiet] | |
info "Done." |