Skip to content
Permalink
Branch: master
Find file Copy path
Find file Copy path
executable file 1776 lines (1547 sloc) 58.9 KB
#: * `test-bot` [options] <url|formula>:
#: Tests the full lifecycle of a formula or Homebrew/brew change.
#:
#: If `--dry-run` is passed, print what would be done rather than doing
#: it.
#:
#: If `--keep-logs` is passed, write and keep log files under
#: `./brewbot/`.
#:
#: If `--cleanup` is passed, clean all state from the Homebrew
#: directory. Use with care!
#:
#: If `--clean-cache` is passed, remove all cached downloads. Use with
#: care!
#:
#: If `--skip-setup` is passed, don't check the local system is setup
#: correctly.
#:
#: If `--skip-homebrew` is passed, don't check Homebrew's files and
#: tests are all valid.
#:
#: If `--junit` is passed, generate a JUnit XML test results file.
#:
#: If `--no-bottle` is passed, run `brew install` without
#: `--build-bottle`.
#:
#: If `--keep-old` is passed, run `brew bottle --keep-old` to build new
#: bottles for a single platform.
#:
#: If `--skip-relocation` is passed, run
#: `brew bottle --skip-relocation` to build new bottles that don't
#: require relocation.
#:
#: If `--HEAD` is passed, run `brew install` with `--HEAD`.
#:
#: If `--local` is passed, ask Homebrew to write verbose logs under
#: `./logs/` and set `$HOME` to `./home/`.
#:
#: If `--tap=<tap>` is passed, use the `git` repository of the given
#: tap.
#:
#: If `--dry-run` is passed, just print commands, don't run them.
#:
#: If `--fail-fast` is passed, immediately exit on a failing step.
#:
#: If `--verbose` (or `-v`) is passed, print test step output in real time. Has
#: the side effect of passing output as raw bytes instead of
#: re-encoding in UTF-8.
#:
#: If `--fast` is passed, don't install any packages, but run e.g.
#: `brew audit` anyway.
#:
#: If `--keep-tmp` is passed, keep temporary files written by main
#: installs and tests that are run.
#:
#: If `--no-pull` is passed, don't use `brew pull` when possible.
#:
#: If `--coverage` is passed, generate and upload a coverage report.
#:
#: If `--test-default-formula` is passed, use a default testing formula
#: when not building a tap and no other formulae are specified.
#:
#: If `--bintray-org=<bintray-org>` is passed, upload to the given Bintray
#: organisation.
#:
#: If `--root-url` is passed, use the specified <URL> as the root of the
#: bottle's URL instead of Homebrew's default.
#:
#: If `--git-name=<git-name>` is passed, set the Git
#: author/committer names to the given name.
#:
#: If `--git-email=<git-email>` is passed, set the Git
#: author/committer email to the given email.
#:
#: If `--ci-master` is passed, use the Homebrew master branch CI
#: options. Implies `--cleanup`: use with care!
#:
#: If `--ci-pr` is passed, use the Homebrew pull request CI options.
#: Implies `--cleanup`: use with care!
#:
#: If `--ci-testing` is passed, use the Homebrew testing CI options.
#: Implies `--cleanup`: use with care!
#:
#: If `--ci-auto` is passed, automatically pick one of the Homebrew CI
#: options based on the environment. Implies `--cleanup`: use with care!
#:
#: If `--ci-upload` is passed, use the Homebrew CI bottle upload
#: options.
require "formula"
require "formula_installer"
require "utils"
require "date"
require "rexml/document"
require "rexml/xmldecl"
require "rexml/cdata"
require "tap"
require "development_tools"
require "utils/bottles"
require "json"
module Homebrew
module_function
BYTES_IN_1_MEGABYTE = 1024*1024
MAX_STEP_OUTPUT_SIZE = BYTES_IN_1_MEGABYTE - (200*1024) # margin of safety
HOMEBREW_TAP_REGEX = %r{^([\w-]+)/homebrew-([\w-]+)$}.freeze
REQUIRED_HOMEBREW_TAPS = [CoreTap.instance.name] + %w[
homebrew/test-bot
].freeze
REQUIRED_LINUXBREW_TAPS = REQUIRED_HOMEBREW_TAPS + %w[
linuxbrew/xorg
].freeze
REQUIRED_TAPS = if OS.mac? || ENV["HOMEBREW_FORCE_HOMEBREW_ON_LINUX"]
REQUIRED_HOMEBREW_TAPS
else
REQUIRED_LINUXBREW_TAPS
end
REQUIRED_TEST_BREW_TAPS = REQUIRED_TAPS + %w[
homebrew/cask
homebrew/bundle
homebrew/services
].freeze
def resolve_test_tap
if (tap = ARGV.value("tap"))
return Tap.fetch(tap)
end
if (tap = ENV["TRAVIS_REPO_SLUG"]) && (tap =~ HOMEBREW_TAP_REGEX)
return Tap.fetch(tap)
end
if ENV["UPSTREAM_BOT_PARAMS"]
bot_argv = ENV["UPSTREAM_BOT_PARAMS"].split(" ")
bot_argv.extend HomebrewArgvExtension
if tap = bot_argv.value("tap")
return Tap.fetch(tap)
end
end
# Get tap from Jenkins UPSTREAM_GIT_URL, GIT_URL or
# Circle CI's CIRCLE_REPOSITORY_URL.
git_url =
ENV["UPSTREAM_GIT_URL"] ||
ENV["GIT_URL"] ||
ENV["CIRCLE_REPOSITORY_URL"] ||
ENV["BUILD_REPOSITORY_URI"]
return unless git_url
url_path = git_url.sub(%r{^https?://.*github\.com/}, "")
.chomp("/")
.sub(/\.git$/, "")
begin
return Tap.fetch(url_path) if url_path =~ HOMEBREW_TAP_REGEX
rescue
# Don't care if tap fetch fails
nil
end
end
# Wraps command invocations. Instantiated by Test#test.
# Handles logging and pretty-printing.
class Step
attr_reader :command, :name, :status, :output, :start_time, :end_time
# Instantiates a Step object.
# @param test [Test] The parent Test object
# @param command [Array<String>] Command to execute and arguments
# @param options [Hash] Recognized options are:
# :repository
# :env
# :puts_output_on_success
def initialize(test, command, repository:, env: {}, puts_output_on_success: false)
@test = test
@category = test.category
@command = command
@puts_output_on_success = puts_output_on_success
@name = command[1].delete("-")
@status = :running
@repository = repository
@env = env
end
def log_file_path
file = "#{@category}.#{@name}.txt"
root = @test.log_root
return file unless root
root + file
end
def command_trimmed
@command.reject { |arg| arg.to_s.start_with?("--exclude") }
.join(" ")
.gsub("#{HOMEBREW_LIBRARY}/Taps/", "")
.gsub("#{HOMEBREW_PREFIX}/", "")
.gsub("/home/travis/", "")
end
def command_short
(@command - %W[
brew
-C
#{HOMEBREW_PREFIX}
#{HOMEBREW_REPOSITORY}
#{@repository}
#{Dir.pwd}
--force
--retry
--verbose
--json
].freeze).join(" ")
.gsub(HOMEBREW_PREFIX.to_s, "")
.gsub(HOMEBREW_REPOSITORY.to_s, "")
.gsub(@repository.to_s, "")
.gsub(Dir.pwd, "")
end
def passed?
@status == :passed
end
def failed?
@status == :failed
end
def self.travis_increment
@travis_step ||= 0
@travis_step += 1
end
def puts_command
puts
puts Formatter.headline(command_trimmed, color: :blue)
end
def puts_result
puts Formatter.headline(Formatter.error("FAILED")) if failed?
end
def output?
@output && !@output.empty?
end
# The execution time of the task.
# Precondition: Step#run has been called.
# @return [Float] execution time in seconds
def time
end_time - start_time
end
def run
@start_time = Time.now
puts_command
if ARGV.include? "--dry-run"
@end_time = Time.now
@status = :passed
puts_result
return
end
if @command[0] == "git" && !%w[-C clone].include?(@command[1])
raise "git should always be called with -C!"
end
executable, *args = @command
verbose = ARGV.verbose?
result = system_command executable, args: args,
print_stdout: verbose,
print_stderr: verbose,
env: @env
@end_time = Time.now
@status = result.success? ? :passed : :failed
puts_result
output = result.merged_output
unless output.empty?
output.force_encoding(Encoding::UTF_8)
@output = if output.valid_encoding?
output
else
output.encode!(Encoding::UTF_16, invalid: :replace)
output.encode!(Encoding::UTF_8)
end
puts @output if (failed? || @puts_output_on_success) && !verbose
File.write(log_file_path, @output) if ARGV.include? "--keep-logs"
end
exit 1 if ARGV.include?("--fail-fast") && failed?
end
end
class Test
attr_reader :log_root, :category, :name, :steps
def initialize(argument, tap: nil, skip_setup: false, skip_homebrew: false, skip_cleanup_before: false, skip_cleanup_after: false)
@hash = nil
@url = nil
@formulae = []
@added_formulae = []
@modified_formulae = []
@deleted_formulae = []
@steps = []
@tap = tap
@repository = if @tap
@test_bot_tap = @tap.to_s == "homebrew/test-bot"
@tap.path
else
HOMEBREW_REPOSITORY
end
@skip_setup = skip_setup
@skip_homebrew = skip_homebrew
@skip_cleanup_before = skip_cleanup_before
@skip_cleanup_after = skip_cleanup_after
if quiet_system("git", "-C", @repository, "rev-parse",
"--verify", "-q", argument)
@hash = argument
elsif url_match = argument.match(HOMEBREW_PULL_OR_COMMIT_URL_REGEX)
@url, _, _, pr = *url_match
@pr_url = @url if pr
elsif canonical_formula_name = safe_formula_canonical_name(argument)
@formulae = [canonical_formula_name]
else
raise ArgumentError,
"#{argument} is not a pull request URL, commit URL or formula name."
end
@category = __method__
@brewbot_root = Pathname.pwd + "brewbot"
FileUtils.mkdir_p @brewbot_root
end
def safe_formula_canonical_name(formula_name)
Formulary.factory(formula_name).full_name
rescue TapFormulaUnavailableError => e
raise if e.tap.installed?
test "brew", "tap", e.tap.name
retry unless steps.last.failed?
onoe e
puts e.backtrace if ARGV.debug?
rescue FormulaUnavailableError, TapFormulaAmbiguityError,
TapFormulaWithOldnameAmbiguityError => e
onoe e
puts e.backtrace if ARGV.debug?
end
def current_sha1
Utils.popen_read("git", "-C", @repository,
"rev-parse", "--short", "HEAD").strip
end
def diff_formulae(start_revision, end_revision, path, filter)
return unless @tap
Utils.popen_read(
"git", "-C", @repository,
"diff-tree", "-r", "--name-only", "--diff-filter=#{filter}",
start_revision, end_revision, "--", path
).lines.map do |line|
file = Pathname.new line.chomp
next unless @tap.formula_file?(file)
@tap.formula_file_to_name(file)
end.compact
end
def merge_commit?(commit)
Utils.popen_read("git", "-C", @repository, "rev-list", "--parents", "-n1", commit).count(" ") > 1
end
def download
@category = __method__
@start_branch = Utils.popen_read(
"git", "-C", @repository, "symbolic-ref", "HEAD"
).gsub("refs/heads/", "").strip
# Use Jenkins GitHub Pull Request Builder or Jenkins Pipeline plugin
# variables for pull request jobs.
if ENV["JENKINS_HOME"] && (ENV["ghprbPullLink"] || ENV["CHANGE_URL"])
@url = ENV["ghprbPullLink"] || ENV["CHANGE_URL"]
@hash = nil
test "git", "-C", @repository, "checkout", "origin/master"
# Use Jenkins Git plugin variables.
elsif ENV["JENKINS_HOME"] && ENV["GIT_URL"] && ENV["GIT_BRANCH"]
git_url = ENV["GIT_URL"].chomp("/").chomp(".git")
%r{origin/pr/(\d+)/(merge|head)} =~ ENV["GIT_BRANCH"]
if pr = Regexp.last_match(1)
@url = "#{git_url}/pull/#{pr}"
@hash = nil
end
# Use Circle CI pull-request variables for pull request jobs.
elsif !ENV["CIRCLE_PULL_REQUEST"].to_s.empty?
@url = ENV["CIRCLE_PULL_REQUEST"]
@hash = nil
# Use Azure Pipeline variables for pull request jobs.
elsif ENV["BUILD_REPOSITORY_URI"] && ENV["SYSTEM_PULLREQUEST_PULLREQUESTNUMBER"]
@url = "#{ENV["BUILD_REPOSITORY_URI"]}/pull/#{ENV["SYSTEM_PULLREQUEST_PULLREQUESTNUMBER"]}"
@hash = nil
end
# Use Jenkins Git plugin variables for master branch jobs.
if ENV["GIT_PREVIOUS_COMMIT"] && ENV["GIT_COMMIT"]
diff_start_sha1 = ENV["GIT_PREVIOUS_COMMIT"]
diff_end_sha1 = ENV["GIT_COMMIT"]
# Use Travis CI Git variables for master or branch jobs.
elsif ENV["TRAVIS_COMMIT_RANGE"]
diff_start_sha1, diff_end_sha1 = ENV["TRAVIS_COMMIT_RANGE"].split "..."
# Use Jenkins Pipeline plugin variables for branch jobs.
elsif ENV["JENKINS_HOME"] && !ENV["CHANGE_URL"] && ENV["CHANGE_TARGET"]
diff_start_sha1 =
Utils.popen_read("git", "-C", @repository, "rev-parse",
"--short", ENV["CHANGE_TARGET"]).strip
diff_end_sha1 = current_sha1
# Use Azure Pipeline variables for master or branch jobs.
elsif ENV["SYSTEM_PULLREQUEST_TARGETBRANCH"] && ENV["BUILD_SOURCEVERSION"]
diff_start_sha1 =
Utils.popen_read("git", "-C", @repository, "rev-parse",
"--short",
ENV["SYSTEM_PULLREQUEST_TARGETBRANCH"]).strip
diff_end_sha1 = current_sha1
# Use CircleCI Git variables.
elsif ENV["CIRCLE_SHA1"]
diff_start_sha1 =
Utils.popen_read("git", "-C", @repository, "rev-parse",
"--short", "origin/master").strip
diff_end_sha1 = ENV["CIRCLE_SHA1"]
# Otherwise just use the current SHA-1 (which may be overriden later)
else
diff_end_sha1 = diff_start_sha1 = current_sha1
end
diff_start_sha1 =
Utils.popen_read("git", "-C", @repository, "merge-base",
diff_start_sha1, diff_end_sha1).strip
# Handle no arguments being passed on the command-line
# e.g. `brew test-bot`
if @hash == "HEAD"
diff_commit_count = Utils.popen_read(
"git", "-C", @repository, "rev-list", "--count",
"#{diff_start_sha1}..#{diff_end_sha1}"
)
@name = if (diff_start_sha1 == diff_end_sha1) ||
(diff_commit_count.to_i == 1)
diff_end_sha1
else
"#{diff_start_sha1}-#{diff_end_sha1}"
end
# Handle formulae arguments being passed on the command-line
# e.g. `brew test-bot wget fish`
elsif !@formulae.empty?
@name = "#{@formulae.first}-#{diff_end_sha1}"
diff_start_sha1 = diff_end_sha1
# Handle a hash being passed on the command-line
# e.g. `brew test-bot 1a2b3c`
elsif @hash
test "git", "-C", @repository, "checkout", @hash
diff_start_sha1 = "#{@hash}^"
diff_end_sha1 = @hash
@name = @hash
# Handle a URL being passed on the command-line or through Jenkins
# environment variables e.g.
# `brew test-bot https://github.com/Homebrew/homebrew-core/pull/678`
elsif @url
unless ARGV.include?("--no-pull")
diff_start_sha1 = current_sha1
test "brew", "pull", "--clean", @url
diff_end_sha1 = current_sha1
end
@short_url = @url.gsub("https://github.com/", "")
@name = if @short_url.include? "/commit/"
# 7 characters should be enough for a commit (not 40).
@short_url.gsub!(%r{(commit/\w{7}).*/}, '\1')
@short_url
else
"#{@short_url}-#{diff_end_sha1}"
end
else
raise "Cannot set @name: invalid command-line arguments!"
end
@log_root = @brewbot_root + @name
FileUtils.mkdir_p @log_root
# Output post-cleanup/download repository revisions.
brew_version = Utils.popen_read(
"git", "-C", HOMEBREW_REPOSITORY.to_s,
"describe", "--tags", "--abbrev", "--dirty"
).strip
brew_commit_subject = Utils.popen_read(
"git", "-C", HOMEBREW_REPOSITORY.to_s,
"log", "-1", "--format=%s"
).strip
puts "Homebrew/brew #{brew_version} (#{brew_commit_subject})"
if @tap.to_s != CoreTap.instance.name
core_path = CoreTap.instance.path
if core_path.exist?
if ARGV.include?("--cleanup")
test "git", "-C", core_path.to_s, "fetch", "--depth=1", "origin"
test "git", "-C", core_path.to_s, "reset", "--hard", "origin/master"
end
else
test "git", "clone", "--depth=1",
CoreTap.instance.default_remote,
core_path.to_s
end
core_revision = Utils.popen_read(
"git", "-C", core_path.to_s,
"log", "-1", "--format=%h (%s)"
).strip
puts "#{CoreTap.instance.full_name} #{core_revision}"
end
if @tap
tap_origin_master_revision = Utils.popen_read(
"git", "-C", @tap.path.to_s,
"log", "-1", "--format=%h (%s)", "origin/master"
).strip
puts "#{@tap} origin/master #{tap_origin_master_revision}"
tap_revision = Utils.popen_read(
"git", "-C", @tap.path.to_s,
"log", "-1", "--format=%h (%s)"
).strip
puts "#{@tap} HEAD #{tap_revision}"
end
return if diff_start_sha1 == diff_end_sha1
return if @url && steps.last && !steps.last.passed?
if @tap && !@test_bot_tap
formula_path = @tap.formula_dir.to_s
@added_formulae +=
diff_formulae(diff_start_sha1, diff_end_sha1, formula_path, "A")
if merge_commit? diff_end_sha1
# Test formulae whose bottles were updated.
summaries = Utils.popen_read("git", "-C", @repository, "log", "--pretty=%s", "#{diff_start_sha1}..#{diff_end_sha1}").lines
@modified_formulae = summaries.map { |s| s[/^([^:]+): update .* bottle\.$/, 1] }.compact.uniq
else
@modified_formulae +=
diff_formulae(diff_start_sha1, diff_end_sha1, formula_path, "M")
end
@deleted_formulae +=
diff_formulae(diff_start_sha1, diff_end_sha1, formula_path, "D")
elsif @formulae.empty? && ARGV.include?("--test-default-formula")
# Build the default test formula.
@test_default_formula = true
@modified_formulae = ["testbottest"]
end
@formulae += @added_formulae + @modified_formulae
@test_brew = (!@tap || @test_bot_tap) &&
(@formulae.empty? || @test_default_formula)
end
def skip(formula_name)
puts Formatter.headline("SKIPPING: #{Formatter.identifier(formula_name)}")
end
def satisfied_requirements?(formula, spec, dependency = nil)
f = Formulary.factory(formula.full_name, spec)
fi = FormulaInstaller.new(f)
stable_spec = spec == :stable
fi.build_bottle = stable_spec && !ARGV.include?("--no-bottle")
unsatisfied_requirements, = fi.expand_requirements
return true if unsatisfied_requirements.empty?
name = formula.full_name
name += " (#{spec})" unless stable_spec
name += " (#{dependency} dependency)" if dependency
skip name
puts unsatisfied_requirements.values.flatten.map(&:message)
false
end
def setup
@category = __method__
return if @skip_setup
# install newer Git when needed
if OS.mac? && MacOS.version < :sierra
test "brew", "install", "git"
ENV["HOMEBREW_FORCE_BREWED_GIT"] = "1"
end
test "brew", "doctor"
test "brew", "--env"
test "brew", "config"
end
def tap_needed_taps(deps)
deps.each { |d| d.to_formula.recursive_dependencies }
rescue TapFormulaUnavailableError => e
raise if e.tap.installed?
e.tap.clear_cache
safe_system "brew", "tap", e.tap.name
retry
end
def install_gcc_if_needed(formula, deps)
installed_gcc = false
begin
deps.each { |dep| CompilerSelector.select_for(dep.to_formula) }
if formula.devel &&
formula.stable? &&
!ARGV.include?("--HEAD") &&
!ARGV.include?("--fast")
CompilerSelector.select_for(formula)
CompilerSelector.select_for(formula.devel)
elsif ARGV.include?("--HEAD")
CompilerSelector.select_for(formula.head)
elsif formula.stable
CompilerSelector.select_for(formula)
elsif formula.devel
CompilerSelector.select_for(formula.devel)
end
rescue CompilerSelectionError => e
unless installed_gcc
test "brew", "install", "gcc",
env: { "HOMEBREW_DEVELOPER" => nil }
installed_gcc = true
DevelopmentTools.clear_version_cache
retry
end
skip formula.name
puts e.message
return
end
end
def install_mercurial_if_needed(deps, reqs)
if (deps | reqs).any? { |d| d.name == "mercurial" && d.build? }
test "brew", "install", "mercurial",
env: { "HOMEBREW_DEVELOPER" => nil }
end
end
def setup_formulae_deps_instances(formula, formula_name)
conflicts = formula.conflicts
formula.recursive_dependencies.each do |dependency|
conflicts += dependency.to_formula.conflicts
end
unlink_formulae = conflicts.map(&:name)
unlink_formulae.uniq.each do |name|
unlink_formula = Formulary.factory(name)
next unless unlink_formula.installed?
next unless unlink_formula.linked_keg.exist?
test "brew", "unlink", name
end
installed = Utils.popen_read("brew", "list").split("\n")
dependencies =
Utils.popen_read("brew", "deps", "--include-build",
"--include-test", formula_name)
.split("\n")
installed_dependencies = installed & dependencies
installed_dependencies.each do |name|
link_formula = Formulary.factory(name)
next if link_formula.keg_only?
next if link_formula.linked_keg.exist?
test "brew", "link", name
end
dependencies -= installed
@unchanged_dependencies = dependencies - @formulae
unless @unchanged_dependencies.empty?
test "brew", "fetch", "--retry", *@unchanged_dependencies
end
changed_dependencies = dependencies - @unchanged_dependencies
unless changed_dependencies.empty?
test "brew", "fetch", "--retry", "--build-from-source",
*changed_dependencies
unless ARGV.include?("--fast")
# Install changed dependencies as new bottles so we don't have
# checksum problems.
test "brew", "install", "--build-from-source", *changed_dependencies
# Run postinstall on them because the tested formula might depend on
# this step
test "brew", "postinstall", *changed_dependencies
end
end
runtime_or_test_dependencies =
Utils.popen_read("brew", "deps", "--include-test", formula_name)
.split("\n")
build_dependencies = dependencies - runtime_or_test_dependencies
@unchanged_build_dependencies = build_dependencies - @formulae
dependents =
Utils.popen_read("brew", "uses", "--include-test", "--recursive", formula_name)
.split("\n")
dependents -= @formulae
dependents = dependents.map { |d| Formulary.factory(d) }
@bottled_dependents = dependents.select(&:bottled?)
@testable_dependents = dependents.select do |d|
d.bottled? && d.test_defined?
end
end
def unlink_conflicts(formula)
return if formula.keg_only?
return if formula.linked_keg.exist?
conflicts = formula.conflicts.map { |c| Formulary.factory(c.name) }
.select(&:installed?)
formula_recursive_dependencies = begin
formula.recursive_dependencies
rescue TapFormulaUnavailableError => e
raise if e.tap.installed?
e.tap.clear_cache
safe_system "brew", "tap", e.tap.name
retry
end
formula_recursive_dependencies.each do |dependency|
conflicts += dependency.to_formula.conflicts.map do |c|
Formulary.factory(c.name)
end.select(&:installed?)
end
conflicts.each do |conflict|
test "brew", "unlink", conflict.name
end
end
def cleanup_bottle_etc_var(formula)
return unless ARGV.include? "--cleanup"
bottle_prefix = formula.opt_prefix/".bottle"
# Nuke etc/var to have them be clean to detect bottle etc/var
# file additions.
Pathname.glob("#{bottle_prefix}/{etc,var}/**/*").each do |bottle_path|
prefix_path = bottle_path.sub(bottle_prefix, HOMEBREW_PREFIX)
FileUtils.rm_rf prefix_path
end
end
def bottle_reinstall_formula(formula, new_formula)
return unless formula.stable?
return if ARGV.include?("--fast")
return if ARGV.include?("--no-bottle")
return if formula.bottle_disabled?
root_url = ARGV.value("root-url")
bottle_args = ["--verbose", "--json", formula.name]
bottle_args << "--keep-old" if ARGV.include?("--keep-old") && !new_formula
bottle_args << "--skip-relocation" if ARGV.include? "--skip-relocation"
bottle_args << "--force-core-tap" if @test_default_formula
bottle_args << "--root-url=#{root_url}" if root_url
bottle_args << "--or-later" if ARGV.include?("--or-later")
test "brew", "bottle", *bottle_args
bottle_step = steps.last
return unless bottle_step.passed?
return unless bottle_step.output?
bottle_filename =
bottle_step.output
.gsub(%r{.*(\./\S+#{Utils::Bottles.native_regex}).*}m, '\1')
bottle_json_filename =
bottle_filename.gsub(/\.(\d+\.)?tar\.gz$/, ".json")
bottle_merge_args =
["--merge", "--write", "--no-commit", bottle_json_filename]
if ARGV.include?("--keep-old") && !new_formula
bottle_merge_args << "--keep-old"
end
test "brew", "bottle", *bottle_merge_args
test "brew", "uninstall", "--force", formula.name
bottle_json = JSON.parse(File.read(bottle_json_filename))
root_url = bottle_json.dig(formula.full_name, "bottle", "root_url")
filename = bottle_json.dig(formula.full_name, "bottle", "tags").values.first["filename"]
download_strategy = CurlDownloadStrategy.new("#{root_url}/#{filename}", formula.name, formula.version)
HOMEBREW_CACHE.mkpath
FileUtils.ln bottle_filename, download_strategy.cached_location, force: true
FileUtils.ln_s download_strategy.cached_location.relative_path_from(download_strategy.symlink_location),
download_strategy.symlink_location,
force: true
@formulae.delete(formula.name)
unless @unchanged_build_dependencies.empty?
test "brew", "uninstall", "--force", *@unchanged_build_dependencies
@unchanged_dependencies -= @unchanged_build_dependencies
end
test "brew", "install", "--only-dependencies", bottle_filename
test "brew", "install", bottle_filename
end
def install_bottled_dependent(dependent)
unless dependent.installed?
test "brew", "fetch", "--retry", dependent.name
return if steps.last.failed?
unlink_conflicts dependent
unless ARGV.include?("--fast")
test "brew", "install", "--only-dependencies", dependent.name,
env: { "HOMEBREW_DEVELOPER" => nil }
test "brew", "install", dependent.name,
env: { "HOMEBREW_DEVELOPER" => nil }
return if steps.last.failed?
end
end
return unless dependent.installed?
if !dependent.keg_only? && !dependent.linked_keg.exist?
unlink_conflicts dependent
test "brew", "link", dependent.name
end
test "brew", "install", "--only-dependencies", dependent.name
test "brew", "linkage", "--test", dependent.name
return unless @testable_dependents.include? dependent
test "brew", "install", "--only-dependencies", "--include-test",
dependent.name
test "brew", "test", "--verbose", dependent.name
end
def fetch_formula(fetch_args, audit_args, spec_args = [])
test "brew", "fetch", "--retry", *spec_args, *fetch_args
test "brew", "audit", *audit_args
end
def formula(formula_name)
@category = "#{__method__}.#{formula_name}"
test "brew", "uses", "--recursive", formula_name
formula = Formulary.factory(formula_name)
deps = []
reqs = []
fetch_args = [formula_name]
if !ARGV.include?("--fast") &&
!ARGV.include?("--no-bottle") &&
!formula.bottle_disabled?
fetch_args << "--build-bottle"
end
fetch_args << "--force" if ARGV.include? "--cleanup"
new_formula = @added_formulae.include?(formula_name)
audit_args = [formula_name, "--online"]
if new_formula
audit_args << "--new-formula"
if url_match = @url.to_s.match(HOMEBREW_PULL_OR_COMMIT_URL_REGEX)
_, _, _, pr = *url_match
ENV["HOMEBREW_NEW_FORMULA_PULL_REQUEST_URL"] = @url if pr
end
end
if formula.stable
unless satisfied_requirements?(formula, :stable)
fetch_formula(fetch_args, audit_args)
return
end
deps |= formula.stable.deps.to_a.reject(&:optional?)
reqs |= formula.stable.requirements.to_a.reject(&:optional?)
elsif formula.devel
unless satisfied_requirements?(formula, :devel)
fetch_formula(fetch_args, audit_args, ["--devel"])
return
end
end
if formula.devel && !ARGV.include?("--HEAD")
deps |= formula.devel.deps.to_a.reject(&:optional?)
reqs |= formula.devel.requirements.to_a.reject(&:optional?)
end
tap_needed_taps(deps)
install_gcc_if_needed(formula, deps)
install_mercurial_if_needed(deps, reqs)
setup_formulae_deps_instances(formula, formula_name)
test "brew", "fetch", "--retry", *fetch_args
test "brew", "uninstall", "--force", formula_name if formula.installed?
# shared_*_args are applied to both the main and --devel spec
shared_install_args = ["--verbose"]
shared_install_args << "--keep-tmp" if ARGV.keep_tmp?
if !ARGV.include?("--fast") &&
!ARGV.include?("--no-bottle") &&
!formula.bottle_disabled?
shared_install_args << "--build-bottle"
end
# install_args is just for the main (stable, or devel if in a devel-only
# tap) spec
install_args = []
install_args << "--HEAD" if ARGV.include? "--HEAD"
# Pass --devel or --HEAD to install in the event formulae lack stable.
# Supports devel-only/head-only.
# head-only should not have devel, but devel-only can have head.
# Stable can have all three.
formula_bottled = if devel_only_tap? formula
install_args << "--devel"
false
elsif head_only_tap? formula
install_args << "--HEAD"
false
else
formula.bottled?
end
install_args += shared_install_args
install_args << formula_name
# Don't care about e.g. bottle failures for dependencies.
install_passed = false
if !ARGV.include?("--fast") || formula_bottled || formula.bottle_unneeded?
test "brew", "install", "--only-dependencies", *install_args,
env: { "HOMEBREW_DEVELOPER" => nil }
test "brew", "install", *install_args,
env: { "HOMEBREW_DEVELOPER" => nil }
install_passed = steps.last.passed?
end
test "brew", "audit", *audit_args
# Only check for style violations if not already shown by
# `brew audit --new-formula`
test "brew", "style", formula_name unless new_formula
test_args = ["--verbose"]
test_args << "--keep-tmp" if ARGV.keep_tmp?
if install_passed
bottle_reinstall_formula(formula, new_formula)
test "brew", "linkage", "--test", formula_name
if formula.test_defined?
test "brew", "install", "--only-dependencies", "--include-test",
formula_name
test "brew", "test", formula_name, *test_args
end
@bottled_dependents.each do |dependent|
install_bottled_dependent(dependent)
end
cleanup_bottle_etc_var(formula)
end
if formula.devel &&
formula.stable? &&
!ARGV.include?("--HEAD") &&
!ARGV.include?("--fast") &&
satisfied_requirements?(formula, :devel)
test "brew", "uninstall", "--force", formula_name if formula.installed?
test "brew", "fetch", "--retry", "--devel", *fetch_args
test "brew", "install", "--devel", "--only-dependencies", formula_name, *shared_install_args,
env: { "HOMEBREW_DEVELOPER" => nil }
test "brew", "install", "--devel", formula_name, *shared_install_args,
env: { "HOMEBREW_DEVELOPER" => nil }
devel_install_passed = steps.last.passed?
if devel_install_passed
test "brew", "postinstall", formula_name
if formula.test_defined?
test "brew", "install", "--devel", "--only-dependencies",
"--include-test", formula_name
test "brew", "test", "--devel", formula_name, *test_args
end
cleanup_bottle_etc_var(formula)
test "brew", "uninstall", "--devel", "--force", formula_name
end
end
unless @all_formulae_dependencies.include?(formula_name)
test "brew", "uninstall", "--force", formula_name
end
return if @unchanged_dependencies.empty?
test "brew", "uninstall", "--force", *@unchanged_dependencies
end
def deleted_formula(formula_name)
@category = "#{__method__}.#{formula_name}"
test "brew", "uses", "--include-build",
"--include-optional",
"--include-test",
formula_name
end
def homebrew
@category = __method__
return if @skip_homebrew
if @test_brew
# test update from origin/master to current commit.
test "brew", "update-test"
# test update from origin/master to current tag.
test "brew", "update-test", "--to-tag"
# test no-op update from current commit (to current commit, a no-op).
test "brew", "update-test", "--commit=HEAD"
installed_taps = Tap.select(&:installed?).map(&:name)
(REQUIRED_TEST_BREW_TAPS - installed_taps).each do |tap|
test "brew", "tap", tap
end
test "brew", "tap", "homebrew/cask"
test "brew", "tap", "homebrew/bundle"
test "brew", "readall", "--aliases"
if OS.mac?
test "brew", "tests", "--no-compat", "--online"
test "brew", "tests", "--generic", "--online"
end
if ARGV.include?("--coverage")
test "brew", "tests", "--online", "--coverage"
FileUtils.cp_r "#{HOMEBREW_REPOSITORY}/Library/Homebrew/test/coverage",
Dir.pwd
else
test "brew", "tests", "--online"
end
# these commands use gems installed by `brew tests`
test "brew", "man", "--fail-if-changed"
test "brew", "style"
elsif @tap
test "brew", "readall", "--aliases", @tap.name
end
end
def cleanup_git_meta(repository)
pr_locks = "#{repository}/.git/refs/remotes/*/pr/*/*.lock"
Dir.glob(pr_locks) { |lock| FileUtils.rm_f lock }
FileUtils.rm_f "#{repository}/.git/gc.log"
end
def clean_if_needed(repository)
return if repository == HOMEBREW_PREFIX
clean_args = [
"-dx",
"--exclude=*.bottle*.*",
"--exclude=coverage",
"--exclude=Library/Taps",
"--exclude=Library/Homebrew/vendor",
"--exclude=#{@brewbot_root.basename}",
]
return if Utils.popen_read(
"git", "-C", repository, "clean", "--dry-run", *clean_args
).strip.empty?
test "git", "-C", repository, "clean", "-ff", *clean_args
end
def prune_if_needed(repository)
return unless Utils.popen_read(
"git -C '#{repository}' -c gc.autoDetach=false gc --auto 2>&1",
).include?("git prune")
test "git", "-C", repository, "prune"
end
def checkout_branch_if_needed(repository, branch = "master")
current_branch = Utils.popen_read(
"git", "-C", repository, "symbolic-ref", "--short", "HEAD"
).strip
return if branch == current_branch
checkout_args = [branch]
checkout_args << "-f" if ARGV.include? "--cleanup"
test "git", "-C", repository, "checkout", *checkout_args
end
def reset_if_needed(repository)
if system("git", "-C", repository, "diff", "--quiet", "origin/master")
return
end
test "git", "-C", repository, "reset", "--hard", "origin/master"
end
def cleanup_shared
cleanup_git_meta(@repository)
clean_if_needed(@repository)
prune_if_needed(@repository)
Tap.names.each do |tap_name|
next if tap_name == @tap&.name
if @test_brew && REQUIRED_TEST_BREW_TAPS.include?(tap_name)
next
elsif REQUIRED_TAPS.include?(tap_name)
next
end
test "brew", "untap", tap_name
end
Keg::MUST_BE_WRITABLE_DIRECTORIES.each(&:mkpath)
Pathname.glob("#{HOMEBREW_PREFIX}/**/*").each do |path|
next if Keg::MUST_BE_WRITABLE_DIRECTORIES.include?(path)
next if path == HOMEBREW_PREFIX/"bin/brew"
next if path == HOMEBREW_PREFIX/"var"
next if path == HOMEBREW_PREFIX/"var/homebrew"
path_string = path.to_s
next if path_string.start_with?(HOMEBREW_REPOSITORY.to_s)
next if path_string.start_with?(@brewbot_root.to_s)
next if path_string.start_with?(Dir.pwd.to_s)
# don't try to delete osxfuse files
next if path_string.match?(
"(include|lib)/(lib|osxfuse/|pkgconfig/)?(osx|mac)?fuse(.*\.(dylib|h|la|pc))?$",
)
FileUtils.rm_rf path
end
if @tap
checkout_branch_if_needed(HOMEBREW_REPOSITORY)
reset_if_needed(HOMEBREW_REPOSITORY)
clean_if_needed(HOMEBREW_REPOSITORY)
end
Pathname.glob("#{HOMEBREW_LIBRARY}/Taps/*/*").each do |git_repo|
cleanup_git_meta(git_repo)
next if @repository == git_repo
checkout_branch_if_needed(git_repo)
reset_if_needed(git_repo)
prune_if_needed(git_repo)
end
end
def clear_stash_if_needed(repository)
return if Utils.popen_read(
"git", "-C", repository, "stash", "list"
).strip.empty?
test "git", "-C", repository, "stash", "clear"
end
def cleanup_before
@category = __method__
return if @skip_cleanup_before
return unless ARGV.include? "--cleanup"
clear_stash_if_needed(@repository)
quiet_system "git", "-C", @repository, "am", "--abort"
quiet_system "git", "-C", @repository, "rebase", "--abort"
unless ARGV.include?("--no-pull")
checkout_branch_if_needed(@repository)
reset_if_needed(@repository)
end
Pathname.glob("*.bottle*.*").each(&:unlink)
# Cleanup NodeJS headers on Azure Pipeline
if OS.linux? && ENV["TF_BUILD"]
test "sudo", "rm", "-rf", "/usr/local/include/node"
end
cleanup_shared
end
def pkill_if_needed!
pgrep = ["pgrep", "-f", HOMEBREW_CELLAR.to_s]
if quiet_system(*pgrep)
test "pkill", "-f", HOMEBREW_CELLAR.to_s
if quiet_system(*pgrep)
sleep 1
test "pkill", "-9", "-f", HOMEBREW_CELLAR.to_s if system(*pgrep)
end
end
end
def cleanup_after
@category = __method__
return if @skip_cleanup_after
return if ENV["CIRCLECI"]
if ENV["HOMEBREW_AZURE_PIPELINES"]
# don't need to do post-build cleanup unless testing test-bot itself.
return if @tap.to_s != "homebrew/test-bot"
end
unless @start_branch.to_s.empty?
checkout_branch_if_needed(@repository, @start_branch)
end
if ARGV.include?("--cleanup")
clear_stash_if_needed(@repository)
reset_if_needed(@repository)
test "brew", "cleanup", "--prune=3"
pkill_if_needed!
cleanup_shared
if ARGV.include? "--local"
FileUtils.rm_rf ENV["HOMEBREW_HOME"]
FileUtils.rm_rf ENV["HOMEBREW_LOGS"]
end
end
FileUtils.rm_rf @brewbot_root unless ARGV.include? "--keep-logs"
end
def test(*args, **options)
step = Step.new(self, args, repository: @repository, **options)
step.run
steps << step
step
end
def check_results
steps.all? do |step|
case step.status
when :passed then true
when :running then raise
when :failed then false
end
end
end
def formulae
changed_formulae_dependents = {}
@all_formulae_dependencies = []
@formulae.each do |formula|
begin
formula_dependencies =
Utils.popen_read("brew", "deps", "--full-name",
"--include-build",
"--include-test", formula)
.split("\n")
# deps can fail if deps are not tapped
unless $CHILD_STATUS.success?
Formulary.factory(formula).recursive_dependencies
end
rescue TapFormulaUnavailableError => e
raise if e.tap.installed?
e.tap.clear_cache
safe_system "brew", "tap", e.tap.name
retry
end
@all_formulae_dependencies += formula_dependencies
unchanged_dependencies = formula_dependencies - @formulae
changed_dependencies = formula_dependencies - unchanged_dependencies
changed_dependencies.each do |changed_formula|
changed_formulae_dependents[changed_formula] ||= 0
changed_formulae_dependents[changed_formula] += 1
end
end
changed_formulae = changed_formulae_dependents.sort do |a1, a2|
a2[1].to_i <=> a1[1].to_i
end
changed_formulae.map!(&:first)
unchanged_formulae = @formulae - changed_formulae
changed_formulae + unchanged_formulae
end
def head_only_tap?(formula)
return false unless formula.head
return false if formula.devel
return false if formula.stable
formula.tap.to_s.downcase !~ %r{[-/]head-only$}
end
def devel_only_tap?(formula)
return false unless formula.devel
return false if formula.stable
formula.tap.to_s.downcase !~ %r{[-/]devel-only$}
end
def run
cleanup_before
begin
download
setup
homebrew
formulae.each do |f|
formula(f)
end
@deleted_formulae.each do |f|
deleted_formula(f)
end
ensure
cleanup_after
end
check_results
end
end
def copy_bottles_from_jenkins
jenkins = ENV["JENKINS_HOME"]
job = ENV["UPSTREAM_JOB_NAME"]
id = ENV["UPSTREAM_BUILD_ID"]
if (!job || !id) && !ARGV.include?("--dry-run")
raise "Missing Jenkins variables!"
end
jenkins_dir = "#{jenkins}/jobs/#{job}/configurations/axis-version/*/"
jenkins_dir += "builds/#{id}/archive/*.bottle*.*"
bottles = Dir[jenkins_dir]
raise "No bottles found in #{jenkins_dir}!" if bottles.empty? && !ARGV.include?("--dry-run")
FileUtils.cp bottles, Dir.pwd, verbose: true
end
def test_ci_upload(tap)
# Don't trust formulae we're uploading
ENV["HOMEBREW_DISABLE_LOAD_FORMULA"] = "1"
bintray_user = ENV["HOMEBREW_BINTRAY_USER"]
bintray_key = ENV["HOMEBREW_BINTRAY_KEY"]
if !bintray_user || !bintray_key
unless ARGV.include?("--dry-run")
raise "Missing HOMEBREW_BINTRAY_USER or HOMEBREW_BINTRAY_KEY variables!"
end
end
# Ensure that uploading Homebrew bottles on Linux doesn't use Linuxbrew.
bintray_org = ARGV.value("bintray-org") || "homebrew"
if bintray_org == "homebrew" && !OS.mac?
ENV["HOMEBREW_FORCE_HOMEBREW_ON_LINUX"] = "1"
end
# Don't pass keys/cookies to subprocesses
ENV.clear_sensitive_environment!
ARGV << "--verbose"
copy_bottles_from_jenkins if !ENV["JENKINS_HOME"].nil?
raise "No bottles found in #{Dir.pwd}!" if Dir["*.bottle*.*"].empty? && !ARGV.include?("--dry-run")
json_files = Dir.glob("*.bottle.json")
bottles_hash = json_files.reduce({}) do |hash, json_file|
hash.deep_merge(JSON.parse(IO.read(json_file)))
end
if ARGV.include?("--dry-run")
bottles_hash = {
"testbottest" => {
"formula" => {
"pkg_version" => "1.0.0",
},
"bottle" => {
"rebuild" => 0,
"tags" => {
Utils::Bottles.tag => {
"filename" =>
"testbottest-1.0.0.#{Utils::Bottles.tag}.bottle.tar.gz",
"sha256" =>
"20cdde424f5fe6d4fdb6a24cff41d2f7aefcd1ef2f98d46f6c074c36a1eef81e",
},
},
},
"bintray" => {
"package" => "testbottest",
"repository" => "bottles",
},
},
}
end
first_formula_name = bottles_hash.keys.first
tap_name = first_formula_name.rpartition("/").first.chuzzle
tap_name ||= CoreTap.instance.name
tap ||= Tap.fetch(tap_name)
ENV["GIT_WORK_TREE"] = tap.path
ENV["GIT_DIR"] = "#{ENV["GIT_WORK_TREE"]}/.git"
ENV["HOMEBREW_GIT_NAME"] = ARGV.value("git-name") || "BrewTestBot"
ENV["HOMEBREW_GIT_EMAIL"] = ARGV.value("git-email") ||
"homebrew-test-bot@lists.sfconservancy.org"
if ARGV.include?("--dry-run")
puts <<~EOS
git am --abort
git rebase --abort
git checkout -f master
git reset --hard origin/master
brew update
EOS
else
quiet_system "git", "am", "--abort"
quiet_system "git", "rebase", "--abort"
safe_system "git", "checkout", "-f", "master"
safe_system "git", "reset", "--hard", "origin/master"
safe_system "brew", "update"
end
# These variables are for Jenkins, Jenkins pipeline and
# Circle CI respectively.
pr = ENV["UPSTREAM_PULL_REQUEST"] ||
ENV["CHANGE_ID"] ||
ENV["CIRCLE_PR_NUMBER"]
if pr
pull_pr = "#{tap.default_remote}/pull/#{pr}"
safe_system "brew", "pull", "--clean", pull_pr
end
if ENV["UPSTREAM_BOTTLE_KEEP_OLD"] ||
ENV["BOT_PARAMS"].to_s.include?("--keep-old") ||
ARGV.include?("--keep-old")
system "brew", "bottle", "--merge", "--write", "--keep-old", *json_files
elsif !ARGV.include?("--dry-run")
system "brew", "bottle", "--merge", "--write", *json_files
else
puts "brew bottle --merge --write $JSON_FILES"
end
# These variables are for Jenkins and Circle CI respectively.
upstream_number = ENV["UPSTREAM_BUILD_NUMBER"] || ENV["CIRCLE_BUILD_NUM"]
git_name = ENV["HOMEBREW_GIT_NAME"]
remote = "git@github.com:#{git_name}/homebrew-#{tap.repo}.git"
git_tag = if pr
"pr-#{pr}"
elsif upstream_number
"testing-#{upstream_number}"
elsif (number = ENV["BUILD_NUMBER"])
"other-#{number}"
elsif ARGV.include?("--dry-run")
"$GIT_TAG"
end
if git_tag
if ARGV.include?("--dry-run")
puts "git push --force #{remote} master:master :refs/tags/#{git_tag}"
else
safe_system "git", "push", "--force", remote,
"master:master", ":refs/tags/#{git_tag}"
end
end
formula_packaged = {}
bottles_hash.each do |formula_name, bottle_hash|
version = bottle_hash["formula"]["pkg_version"]
bintray_package = bottle_hash["bintray"]["package"]
bintray_repo = bottle_hash["bintray"]["repository"]
bintray_packages_url =
"https://api.bintray.com/packages/#{bintray_org}/#{bintray_repo}"
rebuild = bottle_hash["bottle"]["rebuild"]
bottle_hash["bottle"]["tags"].each do |tag, _tag_hash|
filename = Bottle::Filename.new(formula_name, version, tag, rebuild)
bintray_url =
"#{HOMEBREW_BOTTLE_DOMAIN}/#{bintray_repo}/#{filename.bintray}"
filename_already_published = if ARGV.include?("--dry-run")
puts "curl -I --output /dev/null #{bintray_url}"
false
else
begin
system(curl_executable, *curl_args("-I", "--output", "/dev/null",
bintray_url))
end
end
if filename_already_published
raise <<~EOS
#{filename.bintray} is already published. Please remove it manually from
https://bintray.com/#{bintray_org}/#{bintray_repo}/#{bintray_package}/view#files
EOS
end
unless formula_packaged[formula_name]
package_url = "#{bintray_packages_url}/#{bintray_package}"
package_exists = if ARGV.include?("--dry-run")
puts "curl --output /dev/null #{package_url}"
false
else
system(curl_executable, *curl_args("--output", "/dev/null", package_url))
end
unless package_exists
package_blob = <<~EOS
{"name": "#{bintray_package}",
"public_download_numbers": true,
"public_stats": true}
EOS
if ARGV.include?("--dry-run")
puts <<~EOS
curl --user $HOMEBREW_BINTRAY_USER:$HOMEBREW_BINTRAY_KEY
--header Content-Type: application/json
--data #{package_blob.delete("\n")}
#{bintray_packages_url}
EOS
else
curl "--user", "#{bintray_user}:#{bintray_key}",
"--header", "Content-Type: application/json",
"--data", package_blob, bintray_packages_url
puts
end
end
formula_packaged[formula_name] = true
end
content_url = "https://api.bintray.com/content/#{bintray_org}"
content_url +=
"/#{bintray_repo}/#{bintray_package}/#{version}/#{filename.bintray}"
if ARGV.include?("--dry-run")
puts <<~EOS
curl --user $HOMEBREW_BINTRAY_USER:$HOMEBREW_BINTRAY_KEY
--upload-file #{filename}
#{content_url}
EOS
else
curl "--user", "#{bintray_user}:#{bintray_key}",
"--upload-file", filename, content_url
puts
end
end
end
return unless git_tag
if ARGV.include?("--dry-run")
puts "git tag --force #{git_tag}"
puts "git push --force #{remote} master:master refs/tags/#{git_tag}"
else
safe_system "git", "tag", "--force", git_tag
safe_system "git", "push", "--force", remote, "master:master",
"refs/tags/#{git_tag}"
end
end
def sanitize_argv_and_env
if Pathname.pwd == HOMEBREW_PREFIX && ARGV.include?("--cleanup")
odie "cannot use --cleanup from HOMEBREW_PREFIX as it will delete all output."
end
ENV["HOMEBREW_DEVELOPER"] = "1"
ENV["HOMEBREW_NO_AUTO_UPDATE"] = "1"
ENV["HOMEBREW_NO_EMOJI"] = "1"
ENV["HOMEBREW_FAIL_LOG_LINES"] = "150"
ENV["HOMEBREW_PATH"] = ENV["PATH"] =
"#{HOMEBREW_PREFIX}/bin:#{HOMEBREW_PREFIX}/sbin:#{ENV["PATH"]}"
ENV["HOMEBREW_FROZEN_STRING_LITERAL"] = "--enable-frozen-string-literal"
travis = !ENV["TRAVIS"].nil?
circle = !ENV["CIRCLECI"].nil?
if travis || circle
ARGV << "--ci-auto" << "--no-pull"
end
ENV["HOMEBREW_CIRCLECI"] = "1" if circle
if travis
ARGV << "--verbose"
ENV["HOMEBREW_COLOR"] = "1"
ENV["HOMEBREW_VERBOSE_USING_DOTS"] = "1"
ENV["HOMEBREW_TRAVIS_CI"] = "1"
ENV["HOMEBREW_TRAVIS_SUDO"] = ENV["TRAVIS_SUDO"]
end
jenkins = !ENV["JENKINS_HOME"].nil?
jenkins_pipeline_pr = jenkins && !ENV["CHANGE_URL"].nil?
jenkins_pipeline_branch = jenkins &&
!jenkins_pipeline_pr &&
!ENV["BRANCH_NAME"].nil?
ARGV << "--ci-auto" if jenkins_pipeline_branch || jenkins_pipeline_pr
ARGV << "--no-pull" if jenkins_pipeline_branch
azure_pipelines = !ENV["TF_BUILD"].nil?
if azure_pipelines
ARGV << "--verbose" << "--ci-auto" << "--no-pull"
ENV["HOMEBREW_AZURE_PIPELINES"] = "1"
ENV["HOMEBREW_COLOR"] = "1"
# These cannot be queried at the macOS level on Azure Pipelines.
ENV["HOMEBREW_LANGUAGES"] = "en-GB"
end
travis_pr = ENV["TRAVIS_PULL_REQUEST"] &&
ENV["TRAVIS_PULL_REQUEST"] != "false"
jenkins_pr = !ENV["ghprbPullLink"].nil?
jenkins_pr ||= !ENV["ROOT_BUILD_CAUSE_GHPRBCAUSE"].nil?
jenkins_pr ||= jenkins_pipeline_pr
jenkins_branch = !ENV["GIT_COMMIT"].nil?
jenkins_branch ||= jenkins_pipeline_branch
azure_pipelines_pr = ENV["BUILD_REASON"] == "PullRequest"
circle_pr = !ENV["CIRCLE_PULL_REQUEST"].to_s.empty?
# Only report coverage if build runs on macOS and this is indeed Homebrew,
# as we don't want this to be averaged with inferior Linux test coverage.
if OS.mac? && ENV["HOMEBREW_COVERALLS_REPO_TOKEN"]
ARGV << "--coverage"
if azure_pipelines
ENV["HOMEBREW_CI_NAME"] = "azure-pipelines"
ENV["HOMEBREW_CI_BUILD_NUMBER"] = ENV["BUILD_BUILDID"]
ENV["HOMEBREW_CI_BUILD_URL"] = "#{ENV["SYSTEM_TEAMFOUNDATIONSERVERURI"]}#{ENV["SYSTEM_TEAMPROJECT"]}/_build/results?buildId=#{ENV["BUILD_BUILDID"]}"
ENV["HOMEBREW_CI_BRANCH"] = ENV["BUILD_SOURCEBRANCH"]
ENV["HOMEBREW_CI_PULL_REQUEST"] = ENV["SYSTEM_PULLREQUEST_PULLREQUESTNUMBER"]
end
end
if ARGV.include?("--ci-auto")
if travis_pr || jenkins_pr || azure_pipelines_pr || circle_pr
ARGV << "--ci-pr"
elsif travis || jenkins_branch || circle
ARGV << "--ci-master"
else
ARGV << "--ci-testing"
end
end
if ARGV.include?("--ci-master") ||
ARGV.include?("--ci-pr") ||
ARGV.include?("--ci-testing")
ARGV << "--cleanup"
ARGV << "--test-default-formula"
ARGV << "--local" if jenkins
ARGV << "--junit" if jenkins || azure_pipelines || circle
end
ARGV << "--fast" if ARGV.include?("--ci-master")
test_bot_revision = Utils.popen_read(
"git", "-C", Tap.fetch("homebrew/test-bot").path.to_s,
"log", "-1", "--format=%h (%s)"
).strip
puts "Homebrew/homebrew-test-bot #{test_bot_revision}"
puts "ARGV: #{ARGV.join(" ")}"
return unless ARGV.include?("--local")
ENV["HOMEBREW_HOME"] = ENV["HOME"] = "#{Dir.pwd}/home"
mkdir_p ENV["HOMEBREW_HOME"]
ENV["HOMEBREW_LOGS"] = "#{Dir.pwd}/logs"
end
def test_bot
$stdout.sync = true
$stderr.sync = true
sanitize_argv_and_env
tap = resolve_test_tap
# Tap repository if required, this is done before everything else
# because Formula parsing and/or git commit hash lookup depends on it.
# At the same time, make sure Tap is not a shallow clone.
# bottle rebuild and bottle upload rely on full clone.
if tap
if !tap.path.exist?
safe_system "brew", "tap", tap.name, "--full"
elsif (tap.path/".git/shallow").exist?
raise unless quiet_system "git", "-C", tap.path, "fetch", "--unshallow"
end
end
return test_ci_upload(tap) if ARGV.include?("--ci-upload")
tests = []
any_errors = false
skip_setup = ARGV.include?("--skip-setup")
skip_homebrew = ARGV.include?("--skip-homebrew")
skip_cleanup_before = false
if ARGV.named.empty?
# With no arguments just build the most recent commit.
current_test = Test.new("HEAD", tap: tap,
skip_setup: skip_setup,
skip_homebrew: skip_homebrew,
skip_cleanup_before: skip_cleanup_before)
any_errors = !current_test.run
tests << current_test
else
ARGV.named.each do |argument|
skip_cleanup_after = argument != ARGV.named.last
test_error = false
begin
current_test =
Test.new(argument, tap: tap,
skip_setup: skip_setup,
skip_homebrew: skip_homebrew,
skip_cleanup_before: skip_cleanup_before,
skip_cleanup_after: skip_cleanup_after)
skip_setup = true
skip_homebrew = true
skip_cleanup_before = true
rescue ArgumentError => e
test_error = true
ofail e.message
else
test_error = !current_test.run
tests << current_test
end
any_errors ||= test_error
end
end
if ARGV.include? "--junit"
xml_document = REXML::Document.new
xml_document << REXML::XMLDecl.new
testsuites = xml_document.add_element "testsuites"
tests.each do |test|
testsuite = testsuites.add_element "testsuite"
testsuite.add_attribute "name", "brew-test-bot.#{Utils::Bottles.tag}"
testsuite.add_attribute "tests", test.steps.select(&:passed?).count
testsuite.add_attribute "failures", test.steps.select(&:failed?).count
testsuite.add_attribute "timestamp", test.steps.first.start_time.iso8601
test.steps.each do |step|
testcase = testsuite.add_element "testcase"
testcase.add_attribute "name", step.command_short
testcase.add_attribute "status", step.status
testcase.add_attribute "time", step.time
testcase.add_attribute "timestamp", step.start_time.iso8601
next unless step.output?
output = sanitize_output_for_xml(step.output)
cdata = REXML::CData.new output
if step.passed?
elem = testcase.add_element "system-out"
else
elem = testcase.add_element "failure"
elem.add_attribute "message",
"#{step.status}: #{step.command.join(" ")}"
end
elem << cdata
end
end
open("brew-test-bot.xml", "w") do |xml_file|
pretty_print_indent = 2
xml_document.write(xml_file, pretty_print_indent)
end
end
ensure
if HOMEBREW_CACHE.exist?
if ARGV.include? "--clean-cache"
HOMEBREW_CACHE.children.each(&:rmtree)
else
Dir.glob("*.bottle*.tar.gz") do |bottle_file|
FileUtils.rm_f HOMEBREW_CACHE/bottle_file
end
end
end
Homebrew.failed = any_errors
end
def sanitize_output_for_xml(output)
return output if output.empty?
# Remove invalid XML CData characters from step output.
invalid_xml_pat =
/[^\x09\x0A\x0D\x20-\uD7FF\uE000-\uFFFD\u{10000}-\u{10FFFF}]/
output.gsub!(invalid_xml_pat, "\uFFFD")
return output if output.bytesize <= MAX_STEP_OUTPUT_SIZE
# Truncate to 1MB to avoid hitting CI limits
output =
truncate_text_to_approximate_size(
output, MAX_STEP_OUTPUT_SIZE, front_weight: 0.0
)
"truncated output to 1MB:\n#{output}"
end
end
Homebrew.test_bot
You can’t perform that action at this time.