Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Clean up process execution python API #6051

Merged
merged 2 commits into from Jun 29, 2018
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
21 changes: 13 additions & 8 deletions src/python/pants/engine/isolated_process.py
Expand Up @@ -36,17 +36,22 @@ class ExecuteProcessRequest(datatype([
def create_from_snapshot(
cls,
argv,
env,
snapshot,
description,
env=None,
output_files=(),
output_directories=(),
timeout_seconds=_default_timeout_seconds,
description='process'
):
cls._verify_env_is_dict(env)
if env is None:
env = ()
else:
cls._verify_env_is_dict(env)
env = tuple(env.items())

return ExecuteProcessRequest(
argv=argv,
env=tuple(env.items()),
env=env,
input_files=snapshot.directory_digest,
output_files=output_files,
output_directories=output_directories,
Expand All @@ -58,20 +63,20 @@ def create_from_snapshot(
def create_with_empty_snapshot(
cls,
argv,
env,
description,
env=None,
output_files=(),
output_directories=(),
timeout_seconds=_default_timeout_seconds,
description='process'
):
return cls.create_from_snapshot(
argv,
env,
EMPTY_SNAPSHOT,
description,
env,
output_files,
output_directories,
timeout_seconds,
description
)

@classmethod
Expand Down
4 changes: 2 additions & 2 deletions src/python/pants/engine/scheduler.py
Expand Up @@ -14,8 +14,8 @@
from pants.base.exceptions import TaskError
from pants.base.project_tree import Dir, File, Link
from pants.build_graph.address import Address
from pants.engine.fs import (DirectoryDigest, DirectoryToMaterialize,FileContent, FilesContent, Path, PathGlobs,
PathGlobsAndRoot, Snapshot)
from pants.engine.fs import (DirectoryDigest, DirectoryToMaterialize, FileContent, FilesContent,
Path, PathGlobs, PathGlobsAndRoot, Snapshot)
from pants.engine.isolated_process import ExecuteProcessRequest, FallibleExecuteProcessResult
from pants.engine.native import Function, TypeConstraint, TypeId
from pants.engine.nodes import Return, State, Throw
Expand Down
5 changes: 3 additions & 2 deletions src/python/pants/goal/context.py
Expand Up @@ -14,7 +14,7 @@

from pants.base.build_environment import get_buildroot, get_scm
from pants.base.worker_pool import SubprocPool
from pants.base.workunit import WorkUnitLabel
from pants.base.workunit import WorkUnit, WorkUnitLabel
from pants.build_graph.target import Target
from pants.engine.isolated_process import FallibleExecuteProcessResult
from pants.goal.products import Products
Expand Down Expand Up @@ -379,7 +379,7 @@ def scan(self, root=None):
build_graph.inject_address_closure(address)
return build_graph

def execute_process_synchronously(self, execute_process_request, name, labels):
def execute_process_synchronously(self, execute_process_request, name, labels=None):
"""Executes a process (possibly remotely), and returns information about its output.

:param execute_process_request: The ExecuteProcessRequest to run.
Expand All @@ -397,4 +397,5 @@ def execute_process_synchronously(self, execute_process_request, name, labels):
result = self._scheduler.product_request(FallibleExecuteProcessResult, [execute_process_request])[0]
workunit.output("stdout").write(result.stdout)
workunit.output("stderr").write(result.stderr)
workunit.set_outcome(WorkUnit.FAILURE if result.exit_code else WorkUnit.SUCCESS)
return result
28 changes: 10 additions & 18 deletions tests/python/pants_test/engine/test_isolated_process.py
Expand Up @@ -71,9 +71,8 @@ def cat_files_process_result_concatted(cat_exe_req):
cat_files_snapshot = yield Get(Snapshot, PathGlobs, cat_exe_req.path_globs)
process_request = ExecuteProcessRequest.create_from_snapshot(
argv=cat_bin.argv_from_snapshot(cat_files_snapshot),
env=dict(),
snapshot=cat_files_snapshot,
output_files=(),
description='cat some files',
)
cat_process_result = yield Get(ExecuteProcessResult, ExecuteProcessRequest, process_request)
yield Concatted(str(cat_process_result.stdout))
Expand Down Expand Up @@ -102,9 +101,8 @@ def gen_argv(self):
def process_request_from_javac_version(javac_version_exe_req):
yield ExecuteProcessRequest.create_with_empty_snapshot(
argv=javac_version_exe_req.gen_argv(),
env=dict(),
output_files=(),
description=javac_version_exe_req.description)
description=javac_version_exe_req.description,
)


class JavacVersionOutput(datatype([('value', str)])): pass
Expand Down Expand Up @@ -169,7 +167,6 @@ def javac_compile_process_result(javac_compile_req):
output_dirs = tuple({os.path.dirname(java_file) for java_file in java_files})
process_request = ExecuteProcessRequest.create_from_snapshot(
argv=javac_compile_req.argv_from_source_snapshot(sources_snapshot),
env=dict(),
snapshot=sources_snapshot,
output_directories=output_dirs,
description='javac compilation'
Expand Down Expand Up @@ -198,6 +195,7 @@ def _default_args_execute_process_request(self, argv=tuple(), env=None):
env = env or dict()
return ExecuteProcessRequest.create_with_empty_snapshot(
argv=argv,
description='',
env=env,
output_files=(),
)
Expand Down Expand Up @@ -296,9 +294,9 @@ def test_write_file(self):
scheduler = self.mk_scheduler_in_example_fs(())

request = ExecuteProcessRequest.create_with_empty_snapshot(
("/bin/bash", "-c", "echo -n 'European Burmese' > roland"),
dict(),
("roland",)
argv=("/bin/bash", "-c", "echo -n 'European Burmese' > roland"),
description="echo roland",
output_files=("roland",)
)

execute_process_result = self.execute_expecting_one_result(scheduler, ExecuteProcessResult, request).value
Expand Down Expand Up @@ -329,9 +327,7 @@ def test_exercise_python_side_of_timeout_implementation(self):
scheduler = self.mk_scheduler_in_example_fs(())

request = ExecuteProcessRequest.create_with_empty_snapshot(
("/bin/bash", "-c", "/bin/sleep 1; echo -n 'European Burmese'"),
dict(),
tuple(),
argv=("/bin/bash", "-c", "/bin/sleep 1; echo -n 'European Burmese'"),
timeout_seconds=0.1,
description='sleepy-cat',
)
Expand Down Expand Up @@ -378,9 +374,7 @@ def test_fallible_failing_command_returns_exited_result(self):
scheduler = self.mk_scheduler_in_example_fs(())

request = ExecuteProcessRequest.create_with_empty_snapshot(
("/bin/bash", "-c", "exit 1"),
dict(),
tuple(),
argv=("/bin/bash", "-c", "exit 1"),
description='one-cat',
)

Expand All @@ -392,9 +386,7 @@ def test_non_fallible_failing_command_raises(self):
scheduler = self.mk_scheduler_in_example_fs(())

request = ExecuteProcessRequest.create_with_empty_snapshot(
("/bin/bash", "-c", "exit 1"),
dict(),
tuple(),
argv=("/bin/bash", "-c", "exit 1"),
description='one-cat',
)

Expand Down