Skip to content
Permalink
Browse files

Merge branch 'master' of github.com:pantsbuild/pants into py2-wheels-…

…abi-specified
  • Loading branch information...
Eric-Arellano committed Feb 15, 2019
2 parents 1e83f37 + 1ece461 commit f7472aa91b4492ac28d0ada3dddf7ef83258042d
Showing with 1,386 additions and 1,782 deletions.
  1. +3 −1 examples/src/wire/org/pantsbuild/example/element/BUILD
  2. +5 −0 src/python/pants/backend/codegen/wire/java/java_wire_library.py
  3. +34 −9 src/python/pants/backend/codegen/wire/java/wire_gen.py
  4. +1 −1 src/python/pants/backend/graph_info/tasks/cloc.py
  5. +2 −2 src/python/pants/backend/jvm/tasks/jvm_compile/javac/javac_compile.py
  6. +8 −25 src/python/pants/backend/jvm/tasks/jvm_compile/rsc/rsc_compile.py
  7. +17 −22 src/python/pants/backend/jvm/tasks/nailgun_task.py
  8. +6 −29 src/python/pants/backend/native/config/environment.py
  9. +3 −3 src/python/pants/backend/native/subsystems/binaries/gcc.py
  10. +4 −7 src/python/pants/backend/native/subsystems/binaries/llvm.py
  11. +6 −11 src/python/pants/backend/native/subsystems/native_build_step.py
  12. +15 −5 src/python/pants/backend/native/subsystems/native_toolchain.py
  13. +3 −3 src/python/pants/backend/native/targets/native_artifact.py
  14. +3 −3 src/python/pants/backend/native/tasks/conan_fetch.py
  15. +4 −6 src/python/pants/backend/native/tasks/link_shared_libraries.py
  16. +1 −1 src/python/pants/backend/python/rules/python_test_runner.py
  17. +3 −1 src/python/pants/backend/python/subsystems/pytest.py
  18. +3 −3 src/python/pants/backend/python/tasks/unpack_wheels.py
  19. +5 −0 src/python/pants/engine/BUILD
  20. +2 −2 src/python/pants/engine/addressable.py
  21. +1 −1 src/python/pants/engine/build_files.py
  22. +56 −27 src/python/pants/engine/fs.py
  23. +1 −0 src/python/pants/engine/legacy/BUILD
  24. +2 −1 src/python/pants/engine/legacy/graph.py
  25. +0 −8 src/python/pants/engine/native.py
  26. +41 −0 src/python/pants/engine/objects.py
  27. +3 −7 src/python/pants/engine/scheduler.py
  28. +2 −1 src/python/pants/init/engine_initializer.py
  29. +5 −0 src/python/pants/notes/1.14.x.rst
  30. +7 −9 src/python/pants/option/global_options.py
  31. +27 −8 src/python/pants/reporting/reporting.py
  32. +3 −0 src/python/pants/source/filespec.py
  33. +4 −2 src/python/pants/source/wrapped_globs.py
  34. +44 −32 src/python/pants/task/simple_codegen_task.py
  35. +294 −153 src/python/pants/util/objects.py
  36. +92 −26 src/rust/engine/fs/src/snapshot.rs
  37. +121 −104 src/rust/engine/fs/src/store.rs
  38. +14 −12 src/rust/engine/src/lib.rs
  39. +15 −40 src/rust/engine/src/nodes.rs
  40. +0 −4 src/rust/engine/src/types.rs
  41. +0 −3 testprojects/src/resources/org/pantsbuild/testproject/buildfile_path/BUILD
  42. +0 −11 testprojects/src/resources/org/pantsbuild/testproject/ordering/BUILD
  43. 0 testprojects/src/resources/org/pantsbuild/testproject/ordering/a
  44. 0 testprojects/src/resources/org/pantsbuild/testproject/ordering/b
  45. 0 testprojects/src/resources/org/pantsbuild/testproject/ordering/d
  46. 0 testprojects/src/resources/org/pantsbuild/testproject/ordering/i
  47. 0 testprojects/src/resources/org/pantsbuild/testproject/ordering/l
  48. 0 testprojects/src/resources/org/pantsbuild/testproject/ordering/n
  49. 0 testprojects/src/resources/org/pantsbuild/testproject/ordering/p
  50. 0 testprojects/src/resources/org/pantsbuild/testproject/ordering/s
  51. 0 testprojects/src/resources/org/pantsbuild/testproject/ordering/t
  52. 0 testprojects/src/resources/org/pantsbuild/testproject/ordering/u
  53. +1 −24 tests/python/pants_test/BUILD
  54. +10 −2 tests/python/pants_test/backend/codegen/antlr/java/test_antlr_java_gen.py
  55. +1 −0 tests/python/pants_test/backend/codegen/protobuf/java/BUILD
  56. +0 −13 tests/python/pants_test/backend/codegen/protobuf/java/test_protobuf_integration.py
  57. +2 −0 tests/python/pants_test/backend/codegen/wire/java/BUILD
  58. +22 −13 tests/python/pants_test/backend/codegen/wire/java/test_wire_gen.py
  59. +1 −0 tests/python/pants_test/backend/native/tasks/test_cpp_compile.py
  60. +0 −1 tests/python/pants_test/backend/python/tasks/BUILD
  61. +0 −1 tests/python/pants_test/backend/python/tasks/native/BUILD
  62. +53 −57 tests/python/pants_test/backend/python/tasks/native/test_ctypes_integration.py
  63. +0 −504 tests/python/pants_test/base_test.py
  64. +10 −0 tests/python/pants_test/engine/BUILD
  65. +1 −0 tests/python/pants_test/engine/legacy/BUILD
  66. +36 −151 tests/python/pants_test/engine/legacy/test_graph.py
  67. +11 −3 tests/python/pants_test/engine/legacy/test_graph_integration.py
  68. +5 −9 tests/python/pants_test/engine/test_build_files.py
  69. +6 −6 tests/python/pants_test/engine/test_engine.py
  70. +3 −3 tests/python/pants_test/engine/test_fs.py
  71. +2 −4 tests/python/pants_test/engine/test_isolated_process.py
  72. +1 −1 tests/python/pants_test/engine/test_mapper.py
  73. +33 −0 tests/python/pants_test/engine/test_objects.py
  74. +2 −2 tests/python/pants_test/engine/test_scheduler.py
  75. +1 −1 tests/python/pants_test/reporting/BUILD
  76. +84 −0 tests/python/pants_test/reporting/test_reporting.py
  77. +12 −12 tests/python/pants_test/rules/test_test_integration.py
  78. +2 −6 tests/python/pants_test/source/test_payload_fields.py
  79. +1 −5 tests/python/pants_test/source/test_wrapped_globs.py
  80. +12 −4 tests/python/pants_test/task/test_simple_codegen_task.py
  81. +0 −14 tests/python/pants_test/tasks/BUILD
  82. +0 −305 tests/python/pants_test/tasks/task_test_base.py
  83. +7 −7 tests/python/pants_test/test_base.py
  84. +207 −51 tests/python/pants_test/util/test_objects.py
@@ -3,10 +3,12 @@

java_wire_library(
sources=[
'elements.proto', # Order matters here.
# NB: Order matters for these two paths, so we set `ordered_sources=True` below.
'elements.proto',
'compound.proto',
],
dependencies=[
'examples/src/wire/org/pantsbuild/example/temperature',
],
ordered_sources=True,
)
@@ -32,6 +32,7 @@ def __init__(self,
registry_class=None,
enum_options=None,
no_options=None,
ordered_sources=None,
**kwargs):
"""
:param string service_writer: the name of the class to pass as the --service_writer option to
@@ -43,6 +44,9 @@ def __init__(self,
doubt, specify com.squareup.wire.SimpleServiceWriter
:param list enum_options: list of enums to pass to as the --enum-enum_options option, # optional
:param boolean no_options: boolean that determines if --no_options flag is passed
:param boolean ordered_sources: boolean that declares whether the sources argument represents
literal ordered sources to be passed directly to the compiler. If false, no ordering is
guaranteed for the sources passed to an individual compiler invoke.
"""

if not service_writer and service_writer_options:
@@ -59,6 +63,7 @@ def __init__(self,
'registry_class': PrimitiveField(registry_class or None),
'enum_options': PrimitiveField(enum_options or []),
'no_options': PrimitiveField(no_options or False),
'ordered_sources': PrimitiveField(ordered_sources or False),
})

super(JavaWireLibrary, self).__init__(payload=payload, **kwargs)
@@ -13,10 +13,12 @@
from pants.backend.jvm.targets.java_library import JavaLibrary
from pants.backend.jvm.tasks.nailgun_task import NailgunTaskBase
from pants.base.build_environment import get_buildroot
from pants.base.exceptions import TaskError
from pants.base.exceptions import TargetDefinitionException, TaskError
from pants.base.workunit import WorkUnitLabel
from pants.java.jar.jar_dependency import JarDependency
from pants.source.filespec import globs_matches
from pants.task.simple_codegen_task import SimpleCodegenTask
from pants.util.dirutil import fast_relpath


logger = logging.getLogger(__name__)
@@ -61,24 +63,47 @@ def synthetic_target_extra_dependencies(self, target, target_workdir):
wire_runtime_deps_spec = self.get_options().javadeps
return self.resolve_deps([wire_runtime_deps_spec])

def format_args_for_target(self, target, target_workdir):
"""Calculate the arguments to pass to the command line for a single target."""
sources = OrderedSet(target.sources_relative_to_buildroot())

def _compute_sources(self, target):
relative_sources = OrderedSet()
source_roots = set()
for source in sources:
source_roots = OrderedSet()

def capture_and_relativize_to_source_root(source):
source_root = self.context.source_roots.find_by_path(source)
if not source_root:
source_root = self.context.source_roots.find(target)
source_roots.add(source_root.path)
relative_source = os.path.relpath(source, source_root.path)
relative_sources.add(relative_source)
return fast_relpath(source, source_root.path)

if target.payload.get_field_value('ordered_sources'):
# Re-match the filespecs against the sources in order to apply them in the literal order
# they were specified in.
filespec = target.globs_relative_to_buildroot()
excludes = filespec.get('excludes', [])
for filespec in filespec.get('globs', []):
sources = [s for s in target.sources_relative_to_buildroot()
if globs_matches([s], [filespec], excludes)]
if len(sources) != 1:
raise TargetDefinitionException(
target,
'With `ordered_sources=True`, expected one match for each file literal, '
'but got: {} for literal `{}`.'.format(sources, filespec)
)
relative_sources.add(capture_and_relativize_to_source_root(sources[0]))
else:
# Otherwise, use the default (unspecified) snapshot ordering.
for source in target.sources_relative_to_buildroot():
relative_sources.add(capture_and_relativize_to_source_root(source))
return relative_sources, source_roots

def format_args_for_target(self, target, target_workdir):
"""Calculate the arguments to pass to the command line for a single target."""

args = ['--java_out={0}'.format(target_workdir)]

# Add all params in payload to args

relative_sources, source_roots = self._compute_sources(target)

if target.payload.get_field_value('no_options'):
args.append('--no_options')

@@ -40,7 +40,7 @@ def console_output(self, targets):
input_snapshots = tuple(
target.sources_snapshot(scheduler=self.context._scheduler) for target in targets
)
input_files = {f.path for snapshot in input_snapshots for f in snapshot.files}
input_files = {f for snapshot in input_snapshots for f in snapshot.files}

# TODO: Work out a nice library-like utility for writing an argfile, as this will be common.
with temporary_dir() as tmpdir:
@@ -209,8 +209,8 @@ def _execute_hermetic_compile(self, cmd, ctx):
# Assume no extra .class files to grab. We'll fix up that case soon.
# Drop the source_root from the file path.
# Assumes `-d .` has been put in the command.
os.path.relpath(f.path.replace('.java', '.class'), ctx.target.target_base)
for f in input_snapshot.files if f.path.endswith('.java')
os.path.relpath(f.replace('.java', '.class'), ctx.target.target_base)
for f in input_snapshot.files if f.endswith('.java')
)
exec_process_request = ExecuteProcessRequest(
argv=tuple(cmd),
@@ -33,8 +33,7 @@
from pants.java.jar.jar_dependency import JarDependency
from pants.reporting.reporting_utils import items_to_report_element
from pants.util.contextutil import Timer
from pants.util.dirutil import (fast_relpath, fast_relpath_optional, maybe_read_file,
safe_file_dump, safe_mkdir)
from pants.util.dirutil import fast_relpath, fast_relpath_optional, safe_mkdir
from pants.util.memo import memoized_property


@@ -60,22 +59,6 @@ def stdout_contents(wu):
return f.read().rstrip()


def write_digest(output_dir, digest):
safe_file_dump(
'{}.digest'.format(output_dir),
mode='w',
payload='{}:{}'.format(digest.fingerprint, digest.serialized_bytes_length))


def load_digest(output_dir):
read_file = maybe_read_file('{}.digest'.format(output_dir), binary_mode=False)
if read_file:
fingerprint, length = read_file.split(':')
return Digest(fingerprint, int(length))
else:
return None


def _create_desandboxify_fn(possible_path_patterns):
# Takes a collection of possible canonical prefixes, and returns a function that
# if it finds a matching prefix, strips the path prior to the prefix and returns it
@@ -132,7 +115,7 @@ def __init__(self, *args, **kwargs):

@classmethod
def implementation_version(cls):
return super(RscCompile, cls).implementation_version() + [('RscCompile', 171)]
return super(RscCompile, cls).implementation_version() + [('RscCompile', 172)]

@classmethod
def register_options(cls, register):
@@ -201,11 +184,11 @@ def _nailgunnable_combined_classpath(self):

# Overrides the normal zinc compiler classpath, which only contains zinc.
def get_zinc_compiler_classpath(self):
return self.do_for_execution_strategy_variant({
return self.execution_strategy_enum.resolve_for_enum_variant({
self.HERMETIC: lambda: super(RscCompile, self).get_zinc_compiler_classpath(),
self.SUBPROCESS: lambda: super(RscCompile, self).get_zinc_compiler_classpath(),
self.NAILGUN: lambda: self._nailgunnable_combined_classpath,
})
})()

def register_extra_products_from_contexts(self, targets, compile_contexts):
super(RscCompile, self).register_extra_products_from_contexts(targets, compile_contexts)
@@ -218,7 +201,7 @@ def pathglob_for(filename):
def to_classpath_entries(paths, scheduler):
# list of path ->
# list of (path, optional<digest>) ->
path_and_digests = [(p, load_digest(os.path.dirname(p))) for p in paths]
path_and_digests = [(p, Digest.load(os.path.dirname(p))) for p in paths]
# partition: list of path, list of tuples
paths_without_digests = [p for (p, d) in path_and_digests if not d]
if paths_without_digests:
@@ -825,7 +808,7 @@ def _runtool_hermetic(self, main, tool_name, args, distribution, tgt=None, input
raise TaskError(res.stderr)

if output_dir:
write_digest(output_dir, res.output_directory_digest)
res.output_directory_digest.dump(output_dir)
self.context._scheduler.materialize_directories((
DirectoryToMaterialize(
# NB the first element here is the root to materialize into, not the dir to snapshot
@@ -861,15 +844,15 @@ def _runtool_nonhermetic(self, parent_workunit, classpath, main, tool_name, args
def _runtool(self, main, tool_name, args, distribution,
tgt=None, input_files=tuple(), input_digest=None, output_dir=None):
with self.context.new_workunit(tool_name) as wu:
return self.do_for_execution_strategy_variant({
return self.execution_strategy_enum.resolve_for_enum_variant({
self.HERMETIC: lambda: self._runtool_hermetic(
main, tool_name, args, distribution,
tgt=tgt, input_files=input_files, input_digest=input_digest, output_dir=output_dir),
self.SUBPROCESS: lambda: self._runtool_nonhermetic(
wu, self.tool_classpath(tool_name), main, tool_name, args, distribution),
self.NAILGUN: lambda: self._runtool_nonhermetic(
wu, self._nailgunnable_combined_classpath, main, tool_name, args, distribution),
})
})()

def _run_metai_tool(self,
distribution,
@@ -15,6 +15,7 @@
from pants.process.subprocess import Subprocess
from pants.task.task import Task, TaskBase
from pants.util.memo import memoized_property
from pants.util.objects import enum, register_enum_option


class NailgunTaskBase(JvmToolTaskMixin, TaskBase):
@@ -24,30 +25,16 @@ class NailgunTaskBase(JvmToolTaskMixin, TaskBase):
SUBPROCESS = 'subprocess'
HERMETIC = 'hermetic'

class InvalidExecutionStrategyMapping(Exception): pass

_all_execution_strategies = frozenset([NAILGUN, SUBPROCESS, HERMETIC])

def do_for_execution_strategy_variant(self, mapping):
"""Invoke the method in `mapping` with the key corresponding to the execution strategy.
`mapping` is a dict mapping execution strategy -> zero-argument lambda.
"""
variants = frozenset(mapping.keys())
if variants != self._all_execution_strategies:
raise self.InvalidExecutionStrategyMapping(
'Must specify a mapping with exactly the keys {} (was: {})'
.format(self._all_execution_strategies, variants))
method_for_variant = mapping[self.execution_strategy]
# The methods need not return a value, but we pass it along if they do.
return method_for_variant()
class ExecutionStrategy(enum([NAILGUN, SUBPROCESS, HERMETIC])): pass

@classmethod
def register_options(cls, register):
super(NailgunTaskBase, cls).register_options(register)
register('--execution-strategy', choices=[cls.NAILGUN, cls.SUBPROCESS, cls.HERMETIC], default=cls.NAILGUN,
help='If set to nailgun, nailgun will be enabled and repeated invocations of this '
'task will be quicker. If set to subprocess, then the task will be run without nailgun.')
register_enum_option(
register, cls.ExecutionStrategy, '--execution-strategy',
help='If set to nailgun, nailgun will be enabled and repeated invocations of this '
'task will be quicker. If set to subprocess, then the task will be run without nailgun. '
'Hermetic execution is an experimental subprocess execution framework.')
register('--nailgun-timeout-seconds', advanced=True, default=10, type=float,
help='Timeout (secs) for nailgun startup.')
register('--nailgun-connect-attempts', advanced=True, default=5, type=int,
@@ -60,6 +47,13 @@ def register_options(cls, register):
rev='0.9.1'),
])

@memoized_property
def execution_strategy_enum(self):
# TODO: This .create() call can be removed when the enum interface is more stable as the option
# is converted into an instance of self.ExecutionStrategy via the `type` argument through
# register_enum_option().
return self.ExecutionStrategy.create(self.get_options().execution_strategy)

@classmethod
def subsystem_dependencies(cls):
return super(NailgunTaskBase, cls).subsystem_dependencies() + (Subprocess.Factory,)
@@ -76,9 +70,10 @@ def __init__(self, *args, **kwargs):
self._executor_workdir = os.path.join(self.context.options.for_global_scope().pants_workdir,
*id_tuple)

@memoized_property
# TODO: eventually deprecate this when we can move all subclasses to use the enum!
@property
def execution_strategy(self):
return self.get_options().execution_strategy
return self.execution_strategy_enum.value

def create_java_executor(self, dist=None):
"""Create java executor that uses this task's ng daemon, if allowed.
@@ -9,37 +9,14 @@

from pants.engine.rules import SingletonRule
from pants.util.meta import AbstractClass
from pants.util.objects import datatype
from pants.util.objects import datatype, enum
from pants.util.osutil import all_normalized_os_names, get_normalized_os_name
from pants.util.strutil import create_path_env_var


class Platform(datatype(['normalized_os_name'])):
class Platform(enum('normalized_os_name', all_normalized_os_names())):

class UnsupportedPlatformError(Exception):
"""Thrown if pants is running on an unrecognized platform."""

@classmethod
def create(cls):
return Platform(get_normalized_os_name())

_NORMALIZED_OS_NAMES = frozenset(all_normalized_os_names())

def resolve_platform_specific(self, platform_specific_funs):
arg_keys = frozenset(platform_specific_funs.keys())
unknown_plats = self._NORMALIZED_OS_NAMES - arg_keys
if unknown_plats:
raise self.UnsupportedPlatformError(
"platform_specific_funs {} must support platforms {}"
.format(platform_specific_funs, list(unknown_plats)))
extra_plats = arg_keys - self._NORMALIZED_OS_NAMES
if extra_plats:
raise self.UnsupportedPlatformError(
"platform_specific_funs {} has unrecognized platforms {}"
.format(platform_specific_funs, list(extra_plats)))

fun_for_platform = platform_specific_funs[self.normalized_os_name]
return fun_for_platform()
default_value = get_normalized_os_name()


class Executable(AbstractClass):
@@ -89,9 +66,9 @@ def as_invocation_environment_dict(self):
:rtype: dict of string -> string
"""
lib_env_var = self._platform.resolve_platform_specific({
'darwin': lambda: 'DYLD_LIBRARY_PATH',
'linux': lambda: 'LD_LIBRARY_PATH',
lib_env_var = self._platform.resolve_for_enum_variant({
'darwin': 'DYLD_LIBRARY_PATH',
'linux': 'LD_LIBRARY_PATH',
})
return {
'PATH': create_path_env_var(self.path_entries),
@@ -44,9 +44,9 @@ def path_entries(self):

@memoized_method
def _common_lib_dirs(self, platform):
lib64_tuples = platform.resolve_platform_specific({
'darwin': lambda: [],
'linux': lambda: [('lib64',)],
lib64_tuples = platform.resolve_for_enum_variant({
'darwin': [],
'linux': [('lib64',)],
})
return self._filemap(lib64_tuples + [
('lib',),
@@ -80,16 +80,13 @@ def _filemap(self, all_components_list):
def path_entries(self):
return self._filemap([('bin',)])

_PLATFORM_SPECIFIC_LINKER_NAME = {
'darwin': lambda: 'ld64.lld',
'linux': lambda: 'lld',
}

def linker(self, platform):
return Linker(
path_entries=self.path_entries,
exe_filename=platform.resolve_platform_specific(
self._PLATFORM_SPECIFIC_LINKER_NAME),
exe_filename=platform.resolve_for_enum_variant({
'darwin': 'ld64.lld',
'linux': 'lld',
}),
library_dirs=[],
linking_library_dirs=[],
extra_args=[],
Oops, something went wrong.

0 comments on commit f7472aa

Please sign in to comment.
You can’t perform that action at this time.