Skip to content

Commit

Permalink
Ported UnifiedTreeBuilder.
Browse files Browse the repository at this point in the history
  • Loading branch information
@gkistanova
gkistanova committed Oct 27, 2020
1 parent 07ae8be commit 8631fdd
Showing 1 changed file with 65 additions and 57 deletions.
122 changes: 65 additions & 57 deletions zorg/buildbot/builders/UnifiedTreeBuilder.py
View file
@@ -1,6 +1,6 @@
from buildbot.steps.slave import RemoveDirectory
from buildbot.process.properties import WithProperties, Property
from buildbot.steps.shell import SetProperty
from buildbot.plugins import steps, util
from buildbot.steps.shell import SetPropertyFromCommand
from buildbot.process.properties import WithProperties

from zorg.buildbot.commands.CmakeCommand import CmakeCommand
from zorg.buildbot.commands.NinjaCommand import NinjaCommand
Expand Down Expand Up @@ -36,9 +36,9 @@ def cleanBuildRequestedByProperty(step):
**kwargs) # Pass through all the extra arguments.

# Remove the source code for a clean checkout if requested by property.
# TODO: Some Windows slaves do not handle RemoveDirectory command well.
# TODO: Some Windows workers do not handle RemoveDirectory command well.
# So, consider running "rmdir /S /Q <dir>" if the build runs on Windows.
f.addStep(RemoveDirectory(name='clean-src-dir',
f.addStep(steps.RemoveDirectory(name='clean-src-dir',
dir=f.monorepo_dir,
haltOnFailure=False,
flunkOnFailure=False,
Expand Down Expand Up @@ -72,6 +72,7 @@ def addCmakeSteps(
f,
cleanBuildRequested,
obj_dir,
generator=None,
install_dir = None,
extra_configure_args = None,
env = None,
Expand All @@ -89,9 +90,9 @@ def addCmakeSteps(

# This is an incremental build, unless otherwise has been requested.
# Remove obj and install dirs for a clean build.
# TODO: Some Windows slaves do not handle RemoveDirectory command well.
# TODO: Some Windows workers do not handle RemoveDirectory command well.
# So, consider running "rmdir /S /Q <dir>" if the build runs on Windows.
f.addStep(RemoveDirectory(name='clean-%s-dir' % obj_dir,
f.addStep(steps.RemoveDirectory(name='clean-%s-dir' % obj_dir,
dir=obj_dir,
haltOnFailure=False,
flunkOnFailure=False,
Expand All @@ -110,7 +111,7 @@ def addCmakeSteps(
('-DCMAKE_INSTALL_PREFIX=', install_dir_rel),
])

f.addStep(RemoveDirectory(name='clean-%s-dir' % install_dir,
f.addStep(steps.RemoveDirectory(name='clean-%s-dir' % install_dir,
dir=install_dir,
haltOnFailure=False,
flunkOnFailure=False,
Expand All @@ -136,15 +137,27 @@ def addCmakeSteps(

src_dir = LLVMBuildFactory.pathRelativeTo(f.llvm_srcdir, obj_dir)

# Make a local copy of the configure args, as we are going to modify that.
definitions = dict()
options = list()
for d in cmake_args:
if isinstance(d, str) and d.startswith("-D"):
k,v = d[2:].split('=', 1)
definitions[k] = v
else:
options.append(d)

f.addStep(CmakeCommand(name=step_name,
haltOnFailure=True,
description=["Cmake", "configure", stage_name],
options=cmake_args,
path=src_dir,
env=env,
workdir=obj_dir,
**kwargs # Pass through all the extra arguments.
))
haltOnFailure=True,
description=["Cmake", "configure", stage_name],
generator=generator,
definitions=definitions,
options=options,
path=src_dir,
env=env or {},
workdir=obj_dir,
**kwargs # Pass through all the extra arguments.
))

def addNinjaSteps(
f,
Expand All @@ -170,20 +183,27 @@ def addNinjaSteps(
haltOnFailure=True,
targets=targets,
description=["Build", stage_name, "unified", "tree"],
env=env,
env=env or {},
workdir=obj_dir,
**kwargs # Pass through all the extra arguments.
))

# Test just built components if requested.
# Note: At this point env could be None, a dictionary, or a Property object.
if isinstance(env, dict):
check_env = env.copy() if env else dict()
check_env['NINJA_STATUS'] = check_env.get('NINJA_STATUS', "%e [%u/%r/%f] ")
else:
check_env = env or {}

for check in checks:
f.addStep(LitTestCommand(name="test-%s%s" % (step_name, check),
command=['ninja', check],
description=[
"Test", "just", "built", "components", "for",
check,
],
env=env,
env=check_env,
workdir=obj_dir,
**kwargs # Pass through all the extra arguments.
))
Expand All @@ -194,7 +214,7 @@ def addNinjaSteps(
f.addStep(NinjaCommand(name="install-%sall" % step_name,
targets=["install"],
description=["Install", "just", "built", "components"],
env=env,
env=env or {},
workdir=obj_dir,
**kwargs # Pass through all the extra arguments.
))
Expand All @@ -209,14 +229,6 @@ def getCmakeBuildFactory(
env = None,
**kwargs):

# Prepare environmental variables. Set here all env we want everywhere.
merged_env = {
'TERM' : 'dumb' # Be cautious and disable color output from all tools.
}
if env is not None:
# Overwrite pre-set items with the given ones, so user can set anything.
merged_env.update(env)

f = getLLVMBuildFactoryAndSourcecodeSteps(
depends_on_projects=depends_on_projects,
llvm_srcdir=llvm_srcdir,
Expand Down Expand Up @@ -257,6 +269,14 @@ def getCmakeWithNinjaBuildFactory(
if checks is None:
checks = ['check-all']

# Prepare environmental variables. Set here all env we want everywhere.
merged_env = {
'TERM' : 'dumb' # Be cautious and disable color output from all tools.
}
if env:
# Overwrite pre-set items with the given ones, so user can set anything.
merged_env.update(env)

# Some options are required for this build no matter what.
CmakeCommand.applyRequiredOptions(cmake_args, [
('-G', 'Ninja'),
Expand All @@ -269,15 +289,15 @@ def getCmakeWithNinjaBuildFactory(
install_dir=install_dir,
clean=clean,
extra_configure_args=cmake_args,
env=env,
env=merged_env,
**kwargs) # Pass through all the extra arguments.

addNinjaSteps(
f,
obj_dir=f.obj_dir,
checks=checks,
install_dir=f.install_dir,
env=env,
env=merged_env,
**kwargs)

return f
Expand Down Expand Up @@ -308,32 +328,23 @@ def getCmakeWithNinjaWithMSVCBuildFactory(
if checks is None:
checks = ['check-all']

# Set up VS environment, if appropriate.
if not vs:
# We build by Visual Studio 2015, unless otherwise is requested.
vs=r"""%VS140COMNTOOLS%"""

f = getLLVMBuildFactoryAndSourcecodeSteps(
depends_on_projects=depends_on_projects,
llvm_srcdir=llvm_srcdir,
obj_dir=obj_dir,
install_dir=install_dir,
**kwargs) # Pass through all the extra arguments.

f.addStep(SetProperty(
f.addStep(SetPropertyFromCommand(
command=builders_util.getVisualStudioEnvironment(vs, target_arch),
extract_fn=builders_util.extractSlaveEnvironment))
env = Property('slave_env')

# Some options are required for this build no matter what.
CmakeCommand.applyRequiredOptions(cmake_args, [
('-G', 'Ninja'),
])
extract_fn=builders_util.extractVSEnvironment))
env = util.Property('vs_env')

cleanBuildRequested = lambda step: step.build.getProperty("clean", default=step.build.getProperty("clean_obj")) or clean

addCmakeSteps(
f,
generator='Ninja',
cleanBuildRequested=cleanBuildRequested,
obj_dir=f.obj_dir,
install_dir=f.install_dir,
Expand Down Expand Up @@ -405,7 +416,7 @@ def getCmakeWithNinjaMultistageBuildFactory(
llvm_srcdir=llvm_srcdir,
obj_dir=obj_dir,
install_dir=install_dir,
env=env,
env=merged_env,
stage_objdirs=stage_objdirs,
stage_installdirs=stage_installdirs,
stage_names=stage_names,
Expand All @@ -429,11 +440,6 @@ def getCmakeWithNinjaMultistageBuildFactory(
('-DCLANG_BUILD_EXAMPLES=', 'OFF'),
])

# Some options are required for this build no matter what.
CmakeCommand.applyRequiredOptions(cmake_args, [
('-G', 'Ninja'),
])

# The stage 1 is special, though. We use the system compiler and
# do incremental build, unless a clean one has been requested.
cmake_args_stage1 = cmake_args[:]
Expand All @@ -446,11 +452,12 @@ def getCmakeWithNinjaMultistageBuildFactory(

addCmakeSteps(
f,
generator='Ninja',
cleanBuildRequested=cleanBuildRequested,
obj_dir=stage_objdirs[0],
install_dir=stage_installdirs[0],
extra_configure_args=cmake_args_stage1,
env=env,
env=merged_env,
stage_name=stage_names[0],
**kwargs)

Expand All @@ -459,7 +466,7 @@ def getCmakeWithNinjaMultistageBuildFactory(
obj_dir=stage_objdirs[0],
checks=checks,
install_dir=stage_installdirs[0],
env=env,
env=merged_env,
stage_name=stage_names[0],
**kwargs)

Expand Down Expand Up @@ -490,21 +497,22 @@ def getCmakeWithNinjaMultistageBuildFactory(
('-DCMAKE_INSTALL_PREFIX=', install_dir),
])
cmake_args_stageN.append(
WithProperties(
"-DCMAKE_CXX_COMPILER=%(workdir)s/" + staged_install + "/bin/clang++"
util.WithProperties(
"-DCMAKE_CXX_COMPILER=%(builddir)s/" + staged_install + "/bin/clang++"
))
cmake_args_stageN.append(
WithProperties(
"-DCMAKE_C_COMPILER=%(workdir)s/" + staged_install + "/bin/clang"
util.WithProperties(
"-DCMAKE_C_COMPILER=%(builddir)s/" + staged_install + "/bin/clang"
))

addCmakeSteps(
f,
True, # We always do a clean build for the staged builds.
generator='Ninja',
cleanBuildRequested=True, # We always do a clean build for the staged builds.
obj_dir=stage_objdirs[stage_idx],
install_dir=stage_installdirs[stage_idx],
extra_configure_args=cmake_args_stageN,
env=env,
env=merged_env,
stage_name=stage_names[stage_idx],
**kwargs)

Expand All @@ -513,7 +521,7 @@ def getCmakeWithNinjaMultistageBuildFactory(
obj_dir=stage_objdirs[stage_idx],
checks=checks,
install_dir=stage_installdirs[stage_idx],
env=env,
env=merged_env,
stage_name=stage_names[stage_idx],
**kwargs)

Expand Down

0 comments on commit 8631fdd

Please sign in to comment.