Skip to content

Commit

Permalink
Merge #64 by mauriciogg: [SNAP FORK][UPSTREAMED] enable support multi…
Browse files Browse the repository at this point in the history 
…plex workers in busybox actions

#64
Closes #64

COPYBARA_INTEGRATE_REVIEW=#64 from mauriciogg:mgalindo-busybox-workers 761207e
PiperOrigin-RevId: 601199913
Change-Id: I5ecc0b549b405f27fcde3582bf2494ce3c3a9e8d
  • Loading branch information
Mauricio Galindo authored and Copybara-Service committed Jan 24, 2024
1 parent e9df80c commit f53c8c9
Show file tree
Hide file tree
Showing 2 changed files with 82 additions and 19 deletions.
85 changes: 67 additions & 18 deletions rules/busybox.bzl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,12 @@ def _make_resources_flag(
],
)

def _disable_warnings(ctx, args):
if (ctx.fragments.android.persistent_busybox_tools or
ctx.fragments.android.persistent_multiplex_busybox_tools):
# Disable warnings - this are output to stdin/stderr which breaks worker mode
args.add("--logWarnings=false")

def _path(f):
return f.path

Expand Down Expand Up @@ -283,7 +289,8 @@ def _package(
transitive_input_files = []

args = ctx.actions.args()
args.use_param_file("@%s")
args.use_param_file("@%s", use_always = True)
args.set_param_file_format("multiline")
args.add("--tool", "AAPT2_PACKAGE")
args.add("--")
args.add("--aapt2", aapt.executable)
Expand Down Expand Up @@ -395,13 +402,15 @@ def _package(
resource_apks,
join_with = ":",
)

transitive_input_files.append(resource_apks)
_disable_warnings(ctx, args)

_java.run(
_java_run(
ctx = ctx,
host_javabase = host_javabase,
executable = busybox,
tools = [aapt],
tools = [aapt, busybox],
arguments = [args],
inputs = depset(input_files, transitive = transitive_input_files),
outputs = output_files,
Expand All @@ -427,7 +436,8 @@ def _parse(
host_javabase: Target. The host javabase.
"""
args = ctx.actions.args()
args.use_param_file("@%s")
args.use_param_file("@%s", use_always = True)
args.set_param_file_format("multiline")
args.add("--tool", "PARSE")
args.add("--")
args.add(
Expand All @@ -439,7 +449,9 @@ def _parse(
)
args.add("--output", out_symbols)

_java.run(
_disable_warnings(ctx, args)

_java_run(
ctx = ctx,
host_javabase = host_javabase,
executable = busybox,
Expand Down Expand Up @@ -493,7 +505,8 @@ def _merge_assets(
host_javabase: Target. The host javabase.
"""
args = ctx.actions.args()
args.use_param_file("@%s")
args.use_param_file("@%s", use_always = True)
args.set_param_file_format("multiline")
args.add("--tool", "MERGE_ASSETS")
args.add("--")
args.add("--assetsOutput", out_assets_zip)
Expand All @@ -519,10 +532,13 @@ def _merge_assets(
join_with = "&",
)

_java.run(
_disable_warnings(ctx, args)

_java_run(
ctx = ctx,
host_javabase = host_javabase,
executable = busybox,
tools = [busybox],
arguments = [args],
inputs = depset(
assets + [symbols],
Expand Down Expand Up @@ -573,7 +589,8 @@ def _validate_and_link(

# Retrieves the list of files at runtime when a directory is passed.
args = ctx.actions.args()
args.use_param_file("@%s")
args.use_param_file("@%s", use_always = True)
args.set_param_file_format("multiline")
args.add("--tool", "LINK_STATIC_LIBRARY")
args.add("--")
args.add("--aapt2", aapt.executable)
Expand Down Expand Up @@ -604,7 +621,9 @@ def _validate_and_link(
)
input_files.extend(resource_apks)

_java.run(
_disable_warnings(ctx, args)

_java_run(
ctx = ctx,
host_javabase = host_javabase,
executable = busybox,
Expand Down Expand Up @@ -644,7 +663,8 @@ def _compile(

# Retrieves the list of files at runtime when a directory is passed.
args = ctx.actions.args()
args.use_param_file("@%s")
args.use_param_file("@%s", use_always = True)
args.set_param_file_format("multiline")
args.add("--tool", "COMPILE_LIBRARY_RESOURCES")
args.add("--")
args.add("--aapt2", aapt.executable)
Expand All @@ -658,7 +678,9 @@ def _compile(
)
args.add("--output", out_file)

_java.run(
_disable_warnings(ctx, args)

_java_run(
ctx = ctx,
host_javabase = host_javabase,
executable = busybox,
Expand Down Expand Up @@ -720,7 +742,8 @@ def _merge_compiled(
transitive_input_files = []

args = ctx.actions.args()
args.use_param_file("@%s")
args.use_param_file("@%s", use_always = True)
args.set_param_file_format("multiline")
args.add("--tool", "MERGE_COMPILED")
args.add("--")
args.add("--classJarOutput", out_class_jar)
Expand Down Expand Up @@ -759,10 +782,13 @@ def _merge_compiled(
)
transitive_input_files.append(transitive_compiled_resources)

_java.run(
_disable_warnings(ctx, args)

_java_run(
ctx = ctx,
host_javabase = host_javabase,
executable = busybox,
tools = [busybox],
arguments = [args],
inputs = depset(input_files, transitive = transitive_input_files),
outputs = output_files,
Expand All @@ -771,6 +797,12 @@ def _merge_compiled(
"Merging compiled Android Resources in " + out_class_jar.short_path,
)

def _java_run(ctx, mnemonic = None, *args, **kwargs):
enable_workers = ctx.fragments.android.persistent_busybox_tools
multiplex_workers = ctx.fragments.android.persistent_multiplex_busybox_tools

_java.run(ctx, mnemonic = mnemonic, supports_workers = enable_workers, supports_multiplex_workers = multiplex_workers, *args, **kwargs)

def _escape_mv(s):
"""Escapes `:` and `,` in manifest values so they can be used as a busybox flag."""
return s.replace(":", "\\:").replace(",", "\\,")
Expand Down Expand Up @@ -829,6 +861,7 @@ def _merge_manifests(
# Args for busybox
args = ctx.actions.args()
args.use_param_file("@%s", use_always = True)
args.set_param_file_format("multiline")
args.add("--tool", "MERGE_MANIFEST")
args.add("--")
if manifest:
Expand All @@ -851,10 +884,13 @@ def _merge_manifests(
args.add("--log", out_log_file)
outputs.append(out_log_file)

_java.run(
_disable_warnings(ctx, args)

_java_run(
ctx = ctx,
host_javabase = host_javabase,
executable = busybox,
tools = [busybox],
arguments = [args],
inputs = depset(directs, transitive = transitives),
outputs = outputs,
Expand Down Expand Up @@ -894,14 +930,18 @@ def _process_databinding(
res_dirs = _get_unique_res_dirs(resource_files)

args = ctx.actions.args()
args.use_param_file("@%s", use_always = True)
args.set_param_file_format("multiline")
args.add("--tool", "PROCESS_DATABINDING")
args.add("--")
args.add("--output_resource_directory", databinding_resources_dirname)
args.add_all(res_dirs, before_each = "--resource_root")
args.add("--dataBindingInfoOut", out_databinding_info)
args.add("--appId", java_package)

_java.run(
_disable_warnings(ctx, args)

_java_run(
ctx = ctx,
host_javabase = host_javabase,
executable = busybox,
Expand Down Expand Up @@ -948,6 +988,8 @@ def _generate_binary_r(
host_javabase: A Target. The host javabase.
"""
args = ctx.actions.args()
args.use_param_file("@%s", use_always = True)
args.set_param_file_format("multiline")
args.add("--tool", "GENERATE_BINARY_R")
args.add("--")
args.add("--primaryRTxt", r_txt)
Expand All @@ -967,12 +1009,14 @@ def _generate_binary_r(
# TODO(b/154003916): support transitive "--library transitive_r_txt_path,transitive_manifest_path" flags
args.add("--classJarOutput", out_class_jar)
args.add("--targetLabel", str(ctx.label))
args.use_param_file("@%s")

_java.run(
_disable_warnings(ctx, args)

_java_run(
ctx = ctx,
host_javabase = host_javabase,
executable = busybox,
tools = [busybox],
arguments = [args],
inputs = depset([r_txt, manifest], transitive = transitive_r_txts + transitive_manifests),
outputs = [out_class_jar],
Expand Down Expand Up @@ -1012,6 +1056,8 @@ def _make_aar(
when a resource conflict occurs.
"""
args = ctx.actions.args()
args.use_param_file("@%s", use_always = True)
args.set_param_file_format("multiline")
args.add("--tool", "GENERATE_AAR")
args.add("--")
args.add(
Expand All @@ -1031,10 +1077,13 @@ def _make_aar(
if should_throw_on_conflict:
args.add("--throwOnResourceConflict")

_java.run(
_disable_warnings(ctx, args)

_java_run(
ctx = ctx,
host_javabase = host_javabase,
executable = busybox,
tools = [busybox],
arguments = [args],
inputs = (
resource_files +
Expand Down
16 changes: 15 additions & 1 deletion rules/java.bzl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -409,13 +409,21 @@ def _run(
ctx,
host_javabase,
jvm_flags = [],
mnemonic = None,
supports_workers = False,
supports_multiplex_workers = False,
execution_requirements = {},
**args):
"""Run a java binary
Args:
ctx: The context.
host_javabase: Target. The host_javabase.
jvm_flags: Additional arguments to the JVM itself.
mnemonic: A description of the action being executed.
supports_workers: This action is worker-compatible (enabled with --strategy=MyAction=worker).
supports_multiplex_workers: This action is multiplex worker-compatible.
execution_requirements: Information to schedule the action, in key-value pairs.
**args: Additional arguments to pass to ctx.actions.run(). Some will get modified.
"""

Expand Down Expand Up @@ -452,7 +460,13 @@ def _run(

args["arguments"] = jvm_flags + [jar_args] + args.get("arguments", default = [])

ctx.actions.run(**args)
if supports_workers:
execution_requirements["worker-key-mnemonic"] = mnemonic
execution_requirements["supports-workers"] = "1"
if supports_multiplex_workers:
execution_requirements["supports-multiplex-workers"] = "1"

ctx.actions.run(execution_requirements = execution_requirements, mnemonic = mnemonic, **args)

def _create_deploy_jar(
ctx,
Expand Down

0 comments on commit f53c8c9

Please sign in to comment.