Skip to content
Permalink
Browse files

Merge branch 'master' of github.com:pantsbuild/pants into py2-wheels-…

…abi-specified
  • Loading branch information...
Eric-Arellano committed Feb 26, 2019
2 parents 04c4ee0 + 26b0179 commit 78a1aa910c6115aba3c462b5f396f342d96890e3
Showing with 693 additions and 243 deletions.
  1. +1 −1 src/python/pants/VERSION
  2. +78 −59 src/python/pants/backend/jvm/tasks/jvm_compile/zinc/zinc_compile.py
  3. +141 −36 src/python/pants/backend/native/config/environment.py
  4. +3 −2 src/python/pants/backend/native/subsystems/binaries/binutils.py
  5. +2 −2 src/python/pants/backend/native/subsystems/binaries/gcc.py
  6. +2 −2 src/python/pants/backend/native/subsystems/binaries/llvm.py
  7. +1 −0 src/python/pants/backend/native/subsystems/conan.py
  8. +76 −116 src/python/pants/backend/native/subsystems/native_toolchain.py
  9. +5 −4 src/python/pants/backend/native/subsystems/xcode_cli_tools.py
  10. +1 −1 src/python/pants/backend/native/tasks/link_shared_libraries.py
  11. +5 −6 src/python/pants/backend/native/tasks/native_compile.py
  12. +9 −0 src/python/pants/backend/python/subsystems/python_tool_base.py
  13. +32 −4 src/python/pants/backend/python/tasks/python_tool_prep_base.py
  14. +44 −0 src/python/pants/build_graph/target_filter_subsystem.py
  15. +70 −0 src/python/pants/notes/master.rst
  16. +15 −4 src/python/pants/task/task.py
  17. +31 −0 tests/python/pants_test/backend/jvm/tasks/jvm_compile/java/test_zinc_compile_integration.py
  18. +6 −6 tests/python/pants_test/backend/native/subsystems/test_native_toolchain.py
  19. +12 −0 tests/python/pants_test/backend/python/tasks/native/test_ctypes_integration.py
  20. +87 −0 tests/python/pants_test/backend/python/tasks/test_python_tool.py
  21. +11 −0 tests/python/pants_test/build_graph/BUILD
  22. +61 −0 tests/python/pants_test/build_graph/test_target_filter_subsystem.py
@@ -1 +1 @@
1.15.0.dev0
1.15.0.dev1
@@ -386,71 +386,90 @@ def relative_to_exec_root(path):
with open(ctx.zinc_args_file, 'w') as fp:
for arg in zinc_args:
# NB: in Python 2, options are stored sometimes as bytes and sometimes as unicode in the OptionValueContainer.
# This is due to how Python 2 natively stores attributes as a map of `str` (aka `bytes`) to their value. So,
# This is due to how Python 2 natively stores attributes as a map of `str` (aka `bytes`) to their value. So,
# the setattr() and getattr() functions sometimes use bytes.
if PY2:
arg = ensure_text(arg)
fp.write(arg)
fp.write('\n')

if self.execution_strategy == self.HERMETIC:
zinc_relpath = fast_relpath(self._zinc.zinc, get_buildroot())

snapshots = [
self._zinc.snapshot(self.context._scheduler),
ctx.target.sources_snapshot(self.context._scheduler),
]

relevant_classpath_entries = dependency_classpath + [compiler_bridge_classpath_entry]
directory_digests = tuple(
entry.directory_digest for entry in relevant_classpath_entries if entry.directory_digest
)
if len(directory_digests) != len(relevant_classpath_entries):
for dep in relevant_classpath_entries:
if dep.directory_digest is None:
logger.warning(
"ClasspathEntry {} didn't have a Digest, so won't be present for hermetic "
"execution".format(dep)
)

snapshots.extend(
classpath_entry.directory_digest for classpath_entry in scalac_classpath_entries
)

merged_input_digest = self.context._scheduler.merge_directories(
tuple(s.directory_digest for s in (snapshots)) + directory_digests
)

# TODO: Extract something common from Executor._create_command to make the command line
# TODO: Lean on distribution for the bin/java appending here
argv = tuple(['.jdk/bin/java'] + jvm_options + ['-cp', zinc_relpath, Zinc.ZINC_COMPILE_MAIN] + zinc_args)
req = ExecuteProcessRequest(
argv=argv,
input_files=merged_input_digest,
output_directories=(classes_dir,),
description="zinc compile for {}".format(ctx.target.address.spec),
# TODO: These should always be unicodes
# Since this is always hermetic, we need to use `underlying_dist`
jdk_home=text_type(self._zinc.underlying_dist.home),
)
res = self.context.execute_process_synchronously_or_raise(req, self.name(), [WorkUnitLabel.COMPILER])

# TODO: Materialize as a batch in do_compile or somewhere
self.context._scheduler.materialize_directories((
DirectoryToMaterialize(get_buildroot(), res.output_directory_digest),
))

# TODO: This should probably return a ClasspathEntry rather than a Digest
return res.output_directory_digest
else:
if self.runjava(classpath=self.get_zinc_compiler_classpath(),
main=Zinc.ZINC_COMPILE_MAIN,
jvm_options=jvm_options,
args=zinc_args,
workunit_name=self.name(),
workunit_labels=[WorkUnitLabel.COMPILER],
dist=self._zinc.dist):
raise TaskError('Zinc compile failed.')
return self.execution_strategy_enum.resolve_for_enum_variant({
self.HERMETIC: lambda: self._compile_hermetic(
jvm_options, ctx, classes_dir, zinc_args, compiler_bridge_classpath_entry,
dependency_classpath, scalac_classpath_entries),
self.SUBPROCESS: lambda: self._compile_nonhermetic(jvm_options, zinc_args),
self.NAILGUN: lambda: self._compile_nonhermetic(jvm_options, zinc_args),
})()

class ZincCompileError(TaskError):
"""An exception type specifically to signal a failed zinc execution."""

def _compile_nonhermetic(self, jvm_options, zinc_args):
exit_code = self.runjava(classpath=self.get_zinc_compiler_classpath(),
main=Zinc.ZINC_COMPILE_MAIN,
jvm_options=jvm_options,
args=zinc_args,
workunit_name=self.name(),
workunit_labels=[WorkUnitLabel.COMPILER],
dist=self._zinc.dist)
if exit_code != 0:
raise self.ZincCompileError('Zinc compile failed.', exit_code=exit_code)

def _compile_hermetic(self, jvm_options, ctx, classes_dir, zinc_args,
compiler_bridge_classpath_entry, dependency_classpath,
scalac_classpath_entries):
zinc_relpath = fast_relpath(self._zinc.zinc, get_buildroot())

snapshots = [
self._zinc.snapshot(self.context._scheduler),
ctx.target.sources_snapshot(self.context._scheduler),
]

relevant_classpath_entries = dependency_classpath + [compiler_bridge_classpath_entry]
directory_digests = tuple(
entry.directory_digest for entry in relevant_classpath_entries if entry.directory_digest
)
if len(directory_digests) != len(relevant_classpath_entries):
for dep in relevant_classpath_entries:
if dep.directory_digest is None:
logger.warning(
"ClasspathEntry {} didn't have a Digest, so won't be present for hermetic "
"execution".format(dep)
)

snapshots.extend(
classpath_entry.directory_digest for classpath_entry in scalac_classpath_entries
)

# TODO: Extract something common from Executor._create_command to make the command line
# TODO: Lean on distribution for the bin/java appending here
merged_input_digest = self.context._scheduler.merge_directories(
tuple(s.directory_digest for s in snapshots) + directory_digests
)
argv = ['.jdk/bin/java'] + jvm_options + [
'-cp', zinc_relpath,
Zinc.ZINC_COMPILE_MAIN
] + zinc_args

req = ExecuteProcessRequest(
argv=tuple(argv),
input_files=merged_input_digest,
output_directories=(classes_dir,),
description="zinc compile for {}".format(ctx.target.address.spec),
# TODO: These should always be unicodes
# Since this is always hermetic, we need to use `underlying_dist`
jdk_home=text_type(self._zinc.underlying_dist.home),
)
res = self.context.execute_process_synchronously_or_raise(
req, self.name(), [WorkUnitLabel.COMPILER])

# TODO: Materialize as a batch in do_compile or somewhere
self.context._scheduler.materialize_directories((
DirectoryToMaterialize(get_buildroot(), res.output_directory_digest),
))

# TODO: This should probably return a ClasspathEntry rather than a Digest
return res.output_directory_digest

def get_zinc_compiler_classpath(self):
"""Get the classpath for the zinc compiler JVM tool.
Oops, something went wrong.

0 comments on commit 78a1aa9

Please sign in to comment.
You can’t perform that action at this time.