Skip to content
Permalink
master
Switch branches/tags

Name already in use

A tag already exists with the provided branch name. Many Git commands accept both tag and branch names, so creating this branch may cause unexpected behavior. Are you sure you want to create this branch?
Go to file
 
 
Cannot retrieve contributors at this time
executable file 432 lines (355 sloc) 14.1 KB
#!/usr/bin/env ruby
require 'optparse'
require 'yaml'
require 'fileutils'
require 'pathname'
@options = {:num_procs => 2, :memory => 2000}
@bitness = {
'i386' => 32,
'amd64' => 64,
'linux64' => 64,
}
@arches = {
'i386' => 'i386',
'amd64' => 'x86_64',
'linux64' => 'linux64',
}
def system!(cmd)
system(cmd) or raise "failed to run #{cmd}"
end
def sanitize(str, where)
raise "unsanitary string in #{where}" if (str =~ /[^\w.-]/)
str
end
def sanitize_path(str, where)
raise "unsanitary string in #{where}" if (str =~ /[^@\w\/.:+-]/)
str
end
def info(str)
puts str unless @options[:quiet]
end
def build_one_configuration(suite, arch, build_desc)
FileUtils.rm_f("var/install.log")
FileUtils.rm_f("var/build.log")
bits = @bitness[arch] or raise "unknown architecture ${arch}"
if ENV["USE_LXC"] == "1"
ENV["LXC_ARCH"] = arch
ENV["LXC_SUITE"] = suite
end
suitearch = "#{suite}-#{arch}"
info "Stopping target if it is up"
system "stop-target"
sleep 1
unless @options[:skip_image]
info "Making a new image copy"
system! "make-clean-vm --suite #{suite} --arch #{arch}"
end
info "Starting target"
system! "start-target #{bits} #{suitearch}&"
$stdout.write "Checking if target is up"
(1..30).each do
system "on-target true 2> /dev/null" and break
sleep 2
$stdout.write '.'
end
info ''
system! "on-target true"
system! "on-target -u root tee -a /etc/sudoers.d/#{ENV['DISTRO'] || 'ubuntu'} > /dev/null << EOF
%#{ENV['DISTRO'] || 'ubuntu'} ALL=(ALL) NOPASSWD: ALL
EOF" if build_desc["sudo"] and @options[:allow_sudo]
info "Preparing build environment"
system! "on-target setarch #{@arches[arch]} bash < target-bin/init-build.sh"
build_desc["files"].each do |filename|
filename = sanitize(filename, "files section")
system! "copy-to-target #{@quiet_flag} inputs/#{filename} build/"
end
if build_desc["enable_cache"]
if File.directory?("cache/#{build_desc["name"]}")
system! "copy-to-target #{@quiet_flag} cache/#{build_desc["name"]}/ cache/"
end
if File.directory?("cache/common")
system! "copy-to-target #{@quiet_flag} cache/common/ cache/"
end
end
if build_desc["multiarch"]
info "Adding multiarch support (log in var/install.log)"
for a in build_desc["multiarch"]
system! "on-target -u root dpkg --add-architecture #{a} >> var/install.log 2>&1"
end
end
if build_desc["repositories"]
info "Adding repositories to the sources list (log in var/install.log)"
for r in build_desc["repositories"]
system! "on-target -u root tee -a /etc/apt/sources.list >> var/install.log 2>&1 << EOF
#{r["source"]}
EOF"
end
end
info "Updating apt-get repository (log in var/install.log)"
system! "on-target -u root apt-get update >> var/install.log 2>&1"
info "Installing additional packages (log in var/install.log)"
system! "on-target -u root -e DEBIAN_FRONTEND=noninteractive apt-get --no-install-recommends -y install #{build_desc["packages"].join(" ")} >> var/install.log 2>&1"
if build_desc["repositories"]
for r in build_desc["repositories"]
info "Installing additional packages from repository #{r["distribution"]} (log in var/install.log)"
system! "on-target -u root -e DEBIAN_FRONTEND=noninteractive apt-get -t #{r["distribution"]} --no-install-recommends -y install #{r["packages"].join(" ")} >> var/install.log 2>&1"
end
end
if build_desc["alternatives"]
info "Set alternatives (log in var/install.log)"
for a in build_desc["alternatives"]
system! "on-target -u root update-alternatives --set #{a["package"]} #{a["path"]} >> var/install.log 2>&1"
end
end
if @options[:upgrade] || system("on-target -u root '[ ! -e /var/cache/gitian/initial-upgrade ]'")
info "Upgrading system, may take a while (log in var/install.log)"
system! "on-target -u root bash < target-bin/upgrade-system.sh >> var/install.log 2>&1"
end
info "Creating package manifest"
system! "on-target -u root bash < target-bin/grab-packages.sh > var/base-#{suitearch}.manifest"
info "Creating build script (var/build-script)"
File.open("var/build-script", "w") do |script|
script.puts "#!/bin/bash"
script.puts "set -e"
script.puts "export LANG='en_US.UTF-8'"
script.puts "export LC_ALL='en_US.UTF-8'"
script.puts "umask 002"
script.puts "export OUTDIR=$HOME/out"
script.puts "GBUILD_BITS=#{bits}"
if build_desc["enable_cache"]
script.puts "GBUILD_CACHE_ENABLED=1"
script.puts "GBUILD_PACKAGE_CACHE=$HOME/cache/#{build_desc["name"]}"
script.puts "GBUILD_COMMON_CACHE=$HOME/cache/common"
end
script.puts "MAKEOPTS=(-j#{@options[:num_procs]})"
script.puts "NUM_PROCS=#{@options[:num_procs]}"
script.puts "NUM_MEM=#{@options[:memory]}"
script.puts
author_date = nil
build_desc["remotes"].each do |remote|
dir = sanitize(remote["dir"], remote["dir"])
author_date = `cd inputs/#{dir} && TZ=UTC git log --date='format-local:%F %T' --format="%ad" -1`.strip
raise "error looking up author date in #{dir}" unless $?.exitstatus == 0
system! "copy-to-target #{@quiet_flag} inputs/#{dir} build/"
script.puts "(cd build/#{dir} && git reset -q --hard && git clean -q -f -d)"
end
script.puts
ref_datetime = build_desc["reference_datetime"] || author_date
(ref_date, ref_time) = ref_datetime.split
script.puts "REFERENCE_DATETIME='#{ref_datetime}'"
script.puts "REFERENCE_DATE='#{ref_date}'"
script.puts "REFERENCE_TIME='#{ref_time}'"
script.puts
script.puts "cd build"
script.puts build_desc["script"]
end
info "Running build script (log in var/build.log)"
if ENV["CI"]
system! "on-target setarch #{@arches[arch]} bash -x < var/build-script 2>&1 | tee var/build.log"
else
system! "on-target setarch #{@arches[arch]} bash -x < var/build-script > var/build.log 2>&1"
end
end
################################
OptionParser.new do |opts|
opts.banner = "Usage: build [options] <build-description>.yml"
opts.on("--allow-sudo", "override SECURITY on the target VM and allow the use of sudo with no password for the default user") do |v|
@options[:allow_sudo] = v
end
opts.on("-i", "--skip-image", "reuse current target image") do |v|
@options[:skip_image] = v
end
opts.on("--upgrade", "upgrade guest with latest packages") do |v|
@options[:upgrade] = v
end
opts.on("-q", "--quiet", "be quiet") do |v|
@options[:quiet] = v
end
opts.on("-j PROCS", "--num-make PROCS", "number of processes to use") do |v|
@options[:num_procs] = v
end
opts.on("-m MEM", "--memory MEM", "memory to allocate in MiB") do |v|
@options[:memory] = v
end
opts.on("-c PAIRS", "--commit PAIRS", "comma separated list of DIRECTORY=COMMIT pairs") do |v|
@options[:commit] = v
end
opts.on("-u PAIRS", "--url PAIRS", "comma separated list of DIRECTORY=URL pairs") do |v|
@options[:url] = v
end
opts.on("-o", "--cache-read-only", "only use existing cache files, do not update them") do |v|
@options[:cache_ro] = v
end
opts.on("--skip-fetch", "skip fetching the latest git objects and refs from the remote source") do |v|
@options[:skip_fetch] = v
end
opts.on("--fetch-branches", "fetch branches from the remote source") do |v|
@options[:fetch_branches] = v
end
opts.on("--fetch-tags", "fetch tags from the remote source") do |v|
@options[:fetch_tags] = v
end
opts.on("--skip-cleanup", "skip cleaning up the target VM. this may be useful for copying additional files from the target after the build") do |v|
@options[:skip_cleanup] = v
end
end.parse!
if ENV["USE_LXC"] != "1" and ENV["USE_DOCKER"] != "1" and ENV["USE_VBOX"] != "1" and !File.exist?("/dev/kvm")
$stderr.puts "\n************* WARNING: kvm not loaded, this will probably not work out\n\n"
end
base_dir = Pathname.new(__FILE__).expand_path.dirname.parent
libexec_dir = base_dir + 'libexec'
ENV['PATH'] = libexec_dir.to_s + ":" + ENV['PATH']
ENV['GITIAN_BASE'] = base_dir.to_s
ENV['NPROCS'] = @options[:num_procs].to_s
ENV['VMEM'] = @options[:memory].to_s
@quiet_flag = @options[:quiet] ? "-q" : ""
build_desc_file = ARGV.shift or raise "must supply YAML build description file"
build_desc = YAML.load_file(build_desc_file)
in_sums = []
build_dir = 'build'
result_dir = 'result'
cache_dir = 'cache'
enable_cache = build_desc["enable_cache"]
FileUtils.rm_rf(build_dir)
FileUtils.mkdir(build_dir)
FileUtils.mkdir_p(result_dir)
package_name = build_desc["name"] or raise "must supply name"
package_name = sanitize(package_name, "package name")
if enable_cache
FileUtils.mkdir_p(File.join(cache_dir, "common"))
FileUtils.mkdir_p(File.join(cache_dir, package_name))
end
distro = build_desc["distro"] || "ubuntu"
suites = build_desc["suites"] or raise "must supply suites"
archs = build_desc["architectures"] or raise "must supply architectures"
build_desc["reference_datetime"] or build_desc["remotes"].size > 0 or raise "must supply `reference_datetime` or `remotes`"
docker_image_digests = build_desc["docker_image_digests"] || []
# if docker_image_digests are supplied, it must be the same length as suites
if docker_image_digests.size > 0 and suites.size != docker_image_digests.size
raise "`suites` and `docker_image_digests` must both be the same size if both are supplied"
elsif ENV["USE_DOCKER"] == "1" and docker_image_digests.size > 0 and suites.size == docker_image_digests.size
suites = docker_image_digests
end
ENV['DISTRO'] = distro
desc_sum = `sha256sum #{build_desc_file}`
desc_sum = desc_sum.sub(build_desc_file, "#{package_name}-desc.yml")
in_sums << desc_sum
build_desc["files"].each do |filename|
filename = sanitize(filename, "files section")
in_sums << `cd inputs && sha256sum #{filename}`
end
commits = {}
if @options[:commit]
@options[:commit].split(',').each do |pair|
(dir, commit) = pair.split('=')
commits[dir] = commit
end
end
urls = {}
if @options[:url]
@options[:url].split(',').each do |pair|
(dir, url) = pair.split('=')
urls[dir] = url
end
end
build_desc["remotes"].each do |remote|
if !remote["commit"]
remote["commit"] = commits[remote["dir"]]
raise "must specify a commit for directory #{remote["dir"]}" unless remote["commit"]
end
if urls[remote["dir"]]
remote["url"] = urls[remote["dir"]]
end
dir = sanitize(remote["dir"], remote["dir"])
commit = sanitize(remote["commit"], remote["commit"])
unless File.exist?("inputs/#{dir}")
system!("git init inputs/#{dir}")
end
if !@options[:skip_fetch]
if @options[:fetch_branches]
system!("cd inputs/#{dir} && git fetch -f --update-head-ok #{sanitize_path(remote["url"], remote["url"])} +refs/heads/*:refs/heads/*")
end
if @options[:fetch_tags]
system!("cd inputs/#{dir} && git fetch -f --update-head-ok #{sanitize_path(remote["url"], remote["url"])} +refs/tags/*:refs/tags/*")
end
system!("cd inputs/#{dir} && git fetch -f --update-head-ok #{sanitize_path(remote["url"], remote["url"])} #{commit}")
system!("cd inputs/#{dir} && git checkout -q FETCH_HEAD")
else
system!("cd inputs/#{dir} && git checkout -q #{commit}")
end
system!("cd inputs/#{dir} && git submodule update --init --recursive --force")
commit = `cd inputs/#{dir} && git log --format=%H -1`.strip
in_sums << "git:#{commit} #{dir}"
end
base_manifests = YAML::Omap.new
suites.each do |suite|
suite = sanitize(suite, "suite")
archs.each do |arch|
info "--- Building for #{suite} #{arch} ---"
arch = sanitize(arch, "architecture")
# Build!
build_one_configuration(suite, arch, build_desc)
info "Grabbing results from target"
system! "copy-from-target #{@quiet_flag} out #{build_dir}"
if enable_cache && !@options[:cache_ro]
info "Grabbing cache from target"
system! "copy-from-target #{@quiet_flag} cache/#{package_name}/ #{cache_dir}"
system! "copy-from-target #{@quiet_flag} cache/common/ #{cache_dir}"
end
base_manifest = File.read("var/base-#{suite}-#{arch}.manifest")
base_manifests["#{suite}-#{arch}"] = base_manifest
end
end
unless @options[:skip_cleanup]
info "Cleaning up target"
system "stop-target"
end
out_dir = File.join(build_dir, "out")
out_sums = {}
cache_common_dir = File.join(cache_dir, "common")
cache_package_dir = File.join(cache_dir, "#{package_name}")
cache_common_sums = {}
cache_package_sums = {}
info "Generating report"
Dir.glob(File.join(out_dir, '**', '*'), File::FNM_DOTMATCH).sort.each do |file_in_out|
next if File.directory?(file_in_out)
file = file_in_out.sub(out_dir + File::SEPARATOR, '')
file = sanitize_path(file, file_in_out)
out_sums[file] = `cd #{out_dir} && sha256sum #{file}`
raise "failed to sum #{file}" unless $? == 0
puts out_sums[file] unless @options[:quiet]
end
if enable_cache
Dir.glob(File.join(cache_common_dir, '**', '*'), File::FNM_DOTMATCH).sort.each do |file_in_out|
next if File.directory?(file_in_out)
file = file_in_out.sub(cache_common_dir + File::SEPARATOR, '')
file = sanitize_path(file, file_in_out)
cache_common_sums[file] = `cd #{cache_common_dir} && sha256sum #{file}`
raise "failed to sum #{file}" unless $? == 0
end
Dir.glob(File.join(cache_package_dir, '**', '*'), File::FNM_DOTMATCH).sort.each do |file_in_out|
next if File.directory?(file_in_out)
file = file_in_out.sub(cache_package_dir + File::SEPARATOR, '')
file = sanitize_path(file, file_in_out)
cache_package_sums[file] = `cd #{cache_package_dir} && sha256sum #{file}`
raise "failed to sum #{file}" unless $? == 0
end
end
out_manifest = out_sums.keys.sort.map { |key| out_sums[key] }.join('')
in_manifest = in_sums.join('')
cache_common_manifest = cache_common_sums.keys.sort.map { |key| cache_common_sums[key] }.join('')
cache_package_manifest = cache_package_sums.keys.sort.map { |key| cache_package_sums[key] }.join('')
# Use Omap to keep result deterministic
report = YAML::Omap[
'out_manifest', out_manifest,
'in_manifest', in_manifest,
'base_manifests', base_manifests,
'cache_common_manifest', cache_common_manifest,
'cache_package_manifest', cache_package_manifest,
]
result_file = "#{package_name}-res.yml"
File.open(File.join(result_dir, result_file), "w") do |io|
io.write report.to_yaml
end
system!("cd #{result_dir} && sha256sum #{result_file}") unless @options[:quiet]
info "Done."