diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/CompilationSupport.java b/src/main/java/com/google/devtools/build/lib/rules/objc/CompilationSupport.java index 14f60fd0350c9b..a0d1771241eebe 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/objc/CompilationSupport.java +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/CompilationSupport.java @@ -864,7 +864,6 @@ private void registerJ2ObjcDeadCodeRemovalActions( J2ObjcMappingFileProvider j2ObjcMappingFileProvider, J2ObjcEntryClassProvider j2ObjcEntryClassProvider) { NestedSet entryClasses = j2ObjcEntryClassProvider.getEntryClasses(); - Artifact pruner = ruleContext.getPrerequisiteArtifact("$j2objc_dead_code_pruner"); NestedSet j2ObjcDependencyMappingFiles = j2ObjcMappingFileProvider.getDependencyMappingFiles(); NestedSet j2ObjcHeaderMappingFiles = @@ -903,9 +902,8 @@ private void registerJ2ObjcDeadCodeRemovalActions( XcodeConfigInfo.fromRuleContext(ruleContext), appleConfiguration.getSingleArchPlatform()) .setMnemonic("DummyPruner") - .setExecutable(pruner) + .setExecutable(ruleContext.getExecutablePrerequisite("$j2objc_dead_code_pruner")) .addInput(dummyArchive) - .addInput(pruner) .addInput(j2objcArchive) .addInput(xcrunwrapper(ruleContext).getExecutable()) .addTransitiveInputs(j2ObjcDependencyMappingFiles) diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/J2ObjcAspect.java b/src/main/java/com/google/devtools/build/lib/rules/objc/J2ObjcAspect.java index 7621ae9bf879a8..c85f3a5bd42b38 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/objc/J2ObjcAspect.java +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/J2ObjcAspect.java @@ -81,7 +81,6 @@ import com.google.devtools.build.lib.skyframe.ConfiguredTargetAndData; import com.google.devtools.build.lib.skyframe.serialization.autocodec.AutoCodec; import com.google.devtools.build.lib.skyframe.serialization.autocodec.SerializationConstant; -import com.google.devtools.build.lib.util.FileType; import com.google.devtools.build.lib.vfs.PathFragment; import java.io.Serializable; import java.util.ArrayList; @@ -172,22 +171,20 @@ public AspectDefinition getDefinition(AspectParameters aspectParameters) { toolsRepository + "//tools/j2objc:j2objc_deploy.jar"))) .add( attr("$j2objc_wrapper", LABEL) - .allowedFileTypes(FileType.of(".py")) .cfg(ExecutionTransitionFactory.create()) .exec() - .singleArtifact() + .legacyAllowAnyFileType() .value( Label.parseAbsoluteUnchecked( - toolsRepository + "//tools/j2objc:j2objc_wrapper"))) + toolsRepository + "//tools/j2objc:j2objc_wrapper_binary"))) .add( attr("$j2objc_header_map", LABEL) - .allowedFileTypes(FileType.of(".py")) .cfg(ExecutionTransitionFactory.create()) .exec() - .singleArtifact() + .legacyAllowAnyFileType() .value( Label.parseAbsoluteUnchecked( - toolsRepository + "//tools/j2objc:j2objc_header_map"))) + toolsRepository + "//tools/j2objc:j2objc_header_map_binary"))) .add( attr("$jre_emul_jar", LABEL) .cfg(ExecutionTransitionFactory.create()) @@ -584,8 +581,7 @@ private static J2ObjcMappingFileProvider createJ2ObjcTranspilationAction( SpawnAction.Builder transpilationAction = new SpawnAction.Builder() .setMnemonic("TranspilingJ2objc") - .setExecutable(ruleContext.getPrerequisiteArtifact("$j2objc_wrapper")) - .addInput(ruleContext.getPrerequisiteArtifact("$j2objc_wrapper")) + .setExecutable(ruleContext.getExecutablePrerequisite("$j2objc_wrapper")) .addInput(j2ObjcDeployJar) .addInput(bootclasspathJar) .addInputs(moduleFiles) @@ -627,8 +623,7 @@ private static J2ObjcMappingFileProvider createJ2ObjcTranspilationAction( ruleContext.registerAction( new SpawnAction.Builder() .setMnemonic("GenerateJ2objcHeaderMap") - .setExecutable(ruleContext.getPrerequisiteArtifact("$j2objc_header_map")) - .addInput(ruleContext.getPrerequisiteArtifact("$j2objc_header_map")) + .setExecutable(ruleContext.getExecutablePrerequisite("$j2objc_header_map")) .addInputs(sources) .addInputs(sourceJars) .addCommandLine( diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcRuleClasses.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcRuleClasses.java index 2d56a2e43089e1..acb5f26612db13 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcRuleClasses.java +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcRuleClasses.java @@ -664,12 +664,15 @@ public RuleClass build(RuleClass.Builder builder, RuleDefinitionEnvironment env) .direct_compile_time_input() .allowedFileTypes(FileTypeSet.ANY_FILE)) .add( + // This attribute definition must be kept in sync with + // third_party/bazel_rules/rules_apple/apple/internal/rule_factory.bzl attr("$j2objc_dead_code_pruner", LABEL) - .allowedFileTypes(FileType.of(".py")) .cfg(ExecutionTransitionFactory.create()) .exec() - .singleArtifact() - .value(env.getToolsLabel("//tools/objc:j2objc_dead_code_pruner"))) + // Allow arbitrary executable files; this gives more flexibility for the + // implementation of the underlying tool. + .legacyAllowAnyFileType() + .value(env.getToolsLabel("//tools/objc:j2objc_dead_code_pruner_binary"))) .add(attr("$dummy_lib", LABEL).value(env.getToolsLabel("//tools/objc:dummy_lib"))) .build(); } diff --git a/src/test/java/com/google/devtools/build/lib/packages/util/MockJ2ObjcSupport.java b/src/test/java/com/google/devtools/build/lib/packages/util/MockJ2ObjcSupport.java index 7c10ababa54109..91d28074846593 100644 --- a/src/test/java/com/google/devtools/build/lib/packages/util/MockJ2ObjcSupport.java +++ b/src/test/java/com/google/devtools/build/lib/packages/util/MockJ2ObjcSupport.java @@ -17,13 +17,9 @@ import com.google.devtools.build.lib.testutil.TestConstants; import java.io.IOException; -/** - * Creates mock BUILD files required for J2Objc. - */ +/** Creates mock BUILD files required for J2Objc. */ public final class MockJ2ObjcSupport { - /** - * Setup the support for building with J2ObjC. - */ + /** Setup the support for building with J2ObjC. */ public static void setup(MockToolsConfig config) throws IOException { config.create(TestConstants.TOOLS_REPOSITORY_SCRATCH + "third_party/java/j2objc/jre_emul.jar"); config.create(TestConstants.TOOLS_REPOSITORY_SCRATCH + "third_party/java/j2objc/mod/release"); @@ -79,12 +75,14 @@ public static void setup(MockToolsConfig config) throws IOException { TestConstants.LOAD_PROTO_LANG_TOOLCHAIN, "package(default_visibility=['//visibility:public'])", "licenses(['notice'])", - "filegroup(", - " name = 'j2objc_wrapper',", - " srcs = ['j2objc_wrapper.py'])", - "filegroup(", - " name = 'j2objc_header_map',", - " srcs = ['j2objc_header_map.py'])", + "py_binary(", + " name = 'j2objc_wrapper_binary',", + " srcs = ['j2objc_wrapper_binary.py'],", + ")", + "py_binary(", + " name = 'j2objc_header_map_binary',", + " srcs = ['j2objc_header_map_binary.py'],", + ")", "proto_lang_toolchain(", " name = 'j2objc_proto_toolchain',", " blacklisted_protos = [':j2objc_proto_blacklist'],", @@ -94,7 +92,9 @@ public static void setup(MockToolsConfig config) throws IOException { " plugin = '//third_party/java/j2objc:proto_plugin',", " runtime = '//third_party/java/j2objc:proto_runtime',", ")", - "exports_files(['j2objc_deploy.jar'])", + "exports_files([", + " 'j2objc_deploy.jar',", + "])", "proto_library(", " name = 'j2objc_proto_blacklist',", " deps = [", @@ -112,12 +112,15 @@ public static void setup(MockToolsConfig config) throws IOException { if (config.isRealFileSystem()) { config.linkTool(TestConstants.TOOLS_REPOSITORY_SCRATCH + "tools/j2objc/j2objc_deploy.jar"); - config.linkTool(TestConstants.TOOLS_REPOSITORY_SCRATCH + "tools/j2objc/j2objc_wrapper.py"); - config.linkTool(TestConstants.TOOLS_REPOSITORY_SCRATCH + "tools/j2objc/j2objc_header_map.py"); + config.linkTool( + TestConstants.TOOLS_REPOSITORY_SCRATCH + "tools/j2objc/j2objc_wrapper_binary"); + config.linkTool( + TestConstants.TOOLS_REPOSITORY_SCRATCH + "tools/j2objc/j2objc_header_map_binary"); } else { config.create(TestConstants.TOOLS_REPOSITORY_SCRATCH + "tools/j2objc/j2objc_deploy.jar"); - config.create(TestConstants.TOOLS_REPOSITORY_SCRATCH + "tools/j2objc/j2objc_wrapper.py"); - config.create(TestConstants.TOOLS_REPOSITORY_SCRATCH + "tools/j2objc/j2objc_header_map.py"); + config.create(TestConstants.TOOLS_REPOSITORY_SCRATCH + "tools/j2objc/j2objc_wrapper_binary"); + config.create( + TestConstants.TOOLS_REPOSITORY_SCRATCH + "tools/j2objc/j2objc_header_map_binary"); } } } diff --git a/src/test/java/com/google/devtools/build/lib/packages/util/MockObjcSupport.java b/src/test/java/com/google/devtools/build/lib/packages/util/MockObjcSupport.java index 003f0d6e90f579..dc715b2cbde287 100644 --- a/src/test/java/com/google/devtools/build/lib/packages/util/MockObjcSupport.java +++ b/src/test/java/com/google/devtools/build/lib/packages/util/MockObjcSupport.java @@ -168,7 +168,10 @@ public static void setup(MockToolsConfig config) throws IOException { "filegroup(name = 'default_provisioning_profile', srcs = ['foo.mobileprovision'])", "sh_binary(name = 'xcrunwrapper', srcs = ['xcrunwrapper.sh'])", "filegroup(name = 'xctest_infoplist', srcs = ['xctest.plist'])", - "filegroup(name = 'j2objc_dead_code_pruner', srcs = ['j2objc_dead_code_pruner.py'])", + "py_binary(", + " name = 'j2objc_dead_code_pruner_binary',", + " srcs = ['j2objc_dead_code_pruner_binary.py']", + ")", "xcode_config(name = 'host_xcodes',", " default = ':version7_3_1',", " versions = [':version7_3_1', ':version5_0', ':version7_3', ':version5_8', ':version5'])", @@ -208,7 +211,6 @@ public static void setup(MockToolsConfig config) throws IOException { config.create( TestConstants.TOOLS_REPOSITORY_SCRATCH + "tools/objc/foo.mobileprovision", "No such luck"); config.create(TestConstants.TOOLS_REPOSITORY_SCRATCH + "tools/objc/xctest.plist"); - config.create(TestConstants.TOOLS_REPOSITORY_SCRATCH + "tools/objc/j2objc_dead_code_pruner.py"); setupCcToolchainConfig(config); } diff --git a/src/test/java/com/google/devtools/build/lib/rules/objc/BazelJ2ObjcLibraryTest.java b/src/test/java/com/google/devtools/build/lib/rules/objc/BazelJ2ObjcLibraryTest.java index d977ee3567dc0c..1b9f403dcdb40d 100644 --- a/src/test/java/com/google/devtools/build/lib/rules/objc/BazelJ2ObjcLibraryTest.java +++ b/src/test/java/com/google/devtools/build/lib/rules/objc/BazelJ2ObjcLibraryTest.java @@ -529,16 +529,17 @@ public void testJ2ObjcHeaderMappingAction() throws Exception { getConfiguration(target).getBinDirectory(RepositoryName.MAIN).getExecPath() + "/"; assertThat(baseArtifactNames(headerMappingAction.getInputs())) .containsAtLeast("libOne.java", "jar.srcjar"); - assertThat(headerMappingAction.getArguments()) - .containsExactly( - TestConstants.TOOLS_REPOSITORY_PATH_PREFIX + "tools/j2objc/j2objc_header_map.py", - "--source_files", - "java/com/google/transpile/libOne.java", - "--source_jars", - "java/com/google/transpile/jar.srcjar", - "--output_mapping_file", - execPath + "java/com/google/transpile/lib1.mapping.j2objc") - .inOrder(); + assertThat(headerMappingAction.getArguments().get(0)) + .contains("tools/j2objc/j2objc_header_map_binary"); + assertThat(headerMappingAction.getArguments().get(1)).isEqualTo("--source_files"); + assertThat(headerMappingAction.getArguments().get(2)) + .isEqualTo("java/com/google/transpile/libOne.java"); + assertThat(headerMappingAction.getArguments().get(3)).isEqualTo("--source_jars"); + assertThat(headerMappingAction.getArguments().get(4)) + .isEqualTo("java/com/google/transpile/jar.srcjar"); + assertThat(headerMappingAction.getArguments().get(5)).isEqualTo("--output_mapping_file"); + assertThat(headerMappingAction.getArguments().get(6)) + .isEqualTo(execPath + "java/com/google/transpile/lib1.mapping.j2objc"); } protected void checkObjcCompileActions( @@ -1045,17 +1046,17 @@ public void testProtoToolchainForJ2ObjcFlag() throws Exception { TestConstants.LOAD_PROTO_LANG_TOOLCHAIN, "package(default_visibility=['//visibility:public'])", "exports_files(['j2objc_deploy.jar'])", - "filegroup(", - " name = 'j2objc_wrapper',", - " srcs = ['j2objc_wrapper.py'],", + "py_binary(", + " name = 'j2objc_wrapper_binary',", + " srcs = ['j2objc_wrapper_binary.py'],", ")", "proto_library(", " name = 'excluded_protos',", " srcs = ['proto_to_exclude.proto'],", ")", - "filegroup(", - " name = 'j2objc_header_map',", - " srcs = ['j2objc_header_map.py'],", + "py_binary(", + " name = 'j2objc_header_map_binary',", + " srcs = ['j2objc_header_map_binary.py'],", ")", "proto_lang_toolchain(", " name = 'alt_j2objc_proto_toolchain',", @@ -1162,13 +1163,8 @@ public void testJ2ObjcDeadCodeRemovalActionWithOptFlag() throws Exception { "com.google.app.test.test")); SpawnAction deadCodeRemovalAction = (SpawnAction) getGeneratingAction(prunedArchive); - assertContainsSublist( - deadCodeRemovalAction.getArguments(), - new ImmutableList.Builder() - .add( - TestConstants.TOOLS_REPOSITORY_PATH_PREFIX - + "tools/objc/j2objc_dead_code_pruner.py") - .build()); + assertThat(deadCodeRemovalAction.getArguments().get(0)) + .contains("tools/objc/j2objc_dead_code_pruner_binary"); assertThat(deadCodeRemovalAction.getOutputs()).containsExactly(prunedArchive); } diff --git a/src/test/java/com/google/devtools/build/lib/rules/objc/ObjcRuleTestCase.java b/src/test/java/com/google/devtools/build/lib/rules/objc/ObjcRuleTestCase.java index 8a8c8a892322ac..cca510edbc02fc 100644 --- a/src/test/java/com/google/devtools/build/lib/rules/objc/ObjcRuleTestCase.java +++ b/src/test/java/com/google/devtools/build/lib/rules/objc/ObjcRuleTestCase.java @@ -585,7 +585,10 @@ protected static void addAppleBinaryStarlarkRule(Scratch scratch) throws Excepti " default = Label('" + toolsRepo + "//tools/cpp:grep-includes'),", " ),", " '_j2objc_dead_code_pruner': attr.label(", - " default = Label('" + toolsLoc + ":j2objc_dead_code_pruner'),),", + " executable = True,", + " allow_files=True,", + " cfg = 'exec',", + " default = Label('" + toolsLoc + ":j2objc_dead_code_pruner_binary'),),", " '_xcode_config': attr.label(", " default=configuration_field(", " fragment='apple', name='xcode_config_label'),),", diff --git a/tools/j2objc/BUILD.tools b/tools/j2objc/BUILD.tools index 83f43b8a4298ea..da87d38fb7bcbc 100644 --- a/tools/j2objc/BUILD.tools +++ b/tools/j2objc/BUILD.tools @@ -11,6 +11,7 @@ java_binary( runtime_deps = ["@bazel_j2objc//:j2objc"], ) +# TODO(b/225174999): Remove filegroups once blaze update is rolled out. filegroup( name = "j2objc_wrapper", srcs = ["j2objc_wrapper.py"], @@ -21,6 +22,18 @@ filegroup( srcs = ["j2objc_header_map.py"], ) +py_binary( + name = "j2objc_wrapper_binary", + srcs = ["j2objc_wrapper_binary.py"], + python_version = "PY3", +) + +py_binary( + name = "j2objc_header_map_binary", + srcs = ["j2objc_header_map_binary.py"], + python_version = "PY3", +) + proto_lang_toolchain( name = "j2objc_proto_toolchain", blacklisted_protos = [], diff --git a/tools/j2objc/j2objc_header_map_binary.py b/tools/j2objc/j2objc_header_map_binary.py new file mode 100755 index 00000000000000..ecdf1a5dc6118b --- /dev/null +++ b/tools/j2objc/j2objc_header_map_binary.py @@ -0,0 +1,134 @@ +#!/usr/bin/python3 + +# Copyright 2017 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A script to generate Java class to ObjC header mapping for J2ObjC. + +This script generates a text file containing mapping between top-level Java +classes to associated ObjC headers, generated by J2ObjC. + +The mapping file is used by dependent J2ObjC transpilation actions to locate +the correct header import paths for dependent Java classes. + +Inside the script, we read the Java source files and source jars of a single +Java rule, and parse out the package names from the package statements, using +regular expression matching. + +Note that we cannot guarantee 100% correctness by using just regular expression, +but it should be good enough. This allows us to avoid doing any further complex +parsing of the source files and keep the script light-weight without other +dependencies. In the future, we can consider implementing a simple Java lexer +here that correctly parses the package statements out of Java files. +""" + +import argparse +import os +import re +import zipfile + +_PACKAGE_RE = re.compile(r'(package)\s+([\w\.]+);') + + +def _get_file_map_entry(java_file_path, java_file): + """Returns the top-level Java class and header file path tuple. + + Args: + java_file_path: The file path of the source Java file. + java_file: The actual file of the source java file. + Returns: + A tuple containing top-level Java class and associated header file path. Or + None if no package statement exists in the source file. + """ + for line in java_file: + stripped_line = line.strip() + stripped_line_str = stripped_line + if isinstance(stripped_line, bytes): + stripped_line_str = stripped_line.decode('utf-8', 'strict') + elif not isinstance(stripped_line, (str, bytes)): + raise TypeError("not expecting type '%s'" % type(stripped_line_str)) + package_statement = _PACKAGE_RE.search(stripped_line_str) + + # We identified a potential package statement. + if package_statement: + preceding_characters = stripped_line[0:package_statement.start(1)] + # We have preceding characters before the package statement. We need to + # look further into them. + if preceding_characters: + # Skip comment line. + if preceding_characters.startswith('//'): + continue + + # Preceding characters also must end with a space, represent an end + # of comment, or end of a statement. + # Otherwise, we skip the current line. + if not (preceding_characters[len(preceding_characters) - 1].isspace() or + preceding_characters.endswith(';') or + preceding_characters.endswith('*/')): + continue + package_name = package_statement.group(2) + class_name = os.path.splitext(os.path.basename(java_file_path))[0] + header_file = os.path.splitext(java_file_path)[0] + '.h' + return (package_name + '.' + class_name, header_file) + return None + + +def main(): + parser = argparse.ArgumentParser(fromfile_prefix_chars='@') + parser.add_argument( + '--source_files', + required=False, + help='The source files') + parser.add_argument( + '--source_jars', + required=False, + help='The source jars.') + parser.add_argument( + '--output_mapping_file', + required=False, + help='The output mapping file') + + args, _ = parser.parse_known_args() + class_to_header_map = dict() + + # Process the source files. + if args.source_files: + source_files = args.source_files.split(',') + for source_file in source_files: + with open(source_file, 'r') as f: + entry = _get_file_map_entry(source_file, f) + if entry: + class_to_header_map[entry[0]] = entry[1] + + # Process the source jars. + if args.source_jars: + source_jars = args.source_jars.split(',') + for source_jar in source_jars: + with zipfile.ZipFile(source_jar, 'r') as jar: + for jar_entry in jar.namelist(): + if jar_entry.endswith('.java'): + with jar.open(jar_entry) as jar_entry_file: + entry = _get_file_map_entry(jar_entry, jar_entry_file) + if entry: + class_to_header_map[entry[0]] = entry[1] + + # Generate the output header mapping file. + if args.output_mapping_file: + with open(args.output_mapping_file, 'w') as output_mapping_file: + for class_name in sorted(class_to_header_map): + header_path = class_to_header_map[class_name] + output_mapping_file.write(class_name + '=' + header_path + '\n') + +if __name__ == '__main__': + main() diff --git a/tools/j2objc/j2objc_wrapper_binary.py b/tools/j2objc/j2objc_wrapper_binary.py new file mode 100755 index 00000000000000..e6f4c75dbd5c12 --- /dev/null +++ b/tools/j2objc/j2objc_wrapper_binary.py @@ -0,0 +1,538 @@ +#!/usr/bin/python3 + +# Copyright 2015 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A wrapper script for J2ObjC transpiler. + +This script wraps around J2ObjC transpiler to also output a dependency mapping +file by scanning the import and include directives of the J2ObjC-translated +files. +""" + +import argparse +import errno +import multiprocessing +import os +import queue +import re +import shutil +import subprocess +import tempfile +import threading +import zipfile + +_INCLUDE_RE = re.compile('#(include|import) "([^"]+)"') +_CONST_DATE_TIME = [1980, 1, 1, 0, 0, 0] +_ADD_EXPORTS = [ + '--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED', + '--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED', + '--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED', + '--add-exports=jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED', + '--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED', +] + + +def RunJ2ObjC(java, jvm_flags, j2objc, main_class, output_file_path, + j2objc_args, source_paths, files_to_translate): + """Runs J2ObjC transpiler to translate Java source files to ObjC. + + Args: + java: The path of the Java executable. + jvm_flags: A comma-separated list of flags to pass to JVM. + j2objc: The deploy jar of J2ObjC. + main_class: The J2ObjC main class to invoke. + output_file_path: The output file directory. + j2objc_args: A list of args to pass to J2ObjC transpiler. + source_paths: A list of directories that contain sources to translate. + files_to_translate: A list of relative paths (relative to source_paths) that + point to sources to translate. + Returns: + None. + """ + j2objc_args.extend(['-sourcepath', ':'.join(source_paths)]) + j2objc_args.extend(['-d', output_file_path]) + j2objc_args.extend(files_to_translate) + param_file_content = ' '.join(j2objc_args).encode('utf-8') + fd = None + param_filename = None + try: + fd, param_filename = tempfile.mkstemp(text=True) + os.write(fd, param_file_content) + finally: + if fd: + os.close(fd) + try: + j2objc_cmd = [java] + j2objc_cmd.extend([f_ for f_ in jvm_flags.split(',') if f_]) + j2objc_cmd.extend(_ADD_EXPORTS) + j2objc_cmd.extend(['-cp', j2objc, main_class]) + j2objc_cmd.append('@%s' % param_filename) + subprocess.check_call(j2objc_cmd, stderr=subprocess.STDOUT) + finally: + if param_filename: + os.remove(param_filename) + + +def WriteDepMappingFile(objc_files, + objc_file_root, + output_dependency_mapping_file, + file_open=open): + """Scans J2ObjC-translated files and outputs a dependency mapping file. + + The mapping file contains mappings between translated source files and their + imported source files scanned from the import and include directives. + + Args: + objc_files: A list of ObjC files translated by J2ObjC. + objc_file_root: The file path which represents a directory where the + generated ObjC files reside. + output_dependency_mapping_file: The path of the dependency mapping file to + write to. + file_open: Reference to the builtin open function so it may be + overridden for testing. + Raises: + RuntimeError: If spawned threads throw errors during processing. + Returns: + None. + """ + dep_mapping = dict() + input_file_queue = queue.Queue() + output_dep_mapping_queue = queue.Queue() + error_message_queue = queue.Queue() + for objc_file in objc_files: + input_file_queue.put(os.path.join(objc_file_root, objc_file)) + + for _ in range(multiprocessing.cpu_count()): + t = threading.Thread(target=_ReadDepMapping, args=(input_file_queue, + output_dep_mapping_queue, + error_message_queue, + objc_file_root, + file_open)) + t.start() + + input_file_queue.join() + + if not error_message_queue.empty(): + error_messages = list(error_message_queue.queue) + raise RuntimeError('\n'.join(error_messages)) + + while not output_dep_mapping_queue.empty(): + entry_file, deps = output_dep_mapping_queue.get() + dep_mapping[entry_file] = deps + + with file_open(output_dependency_mapping_file, 'w') as f: + for entry in sorted(dep_mapping): + for dep in dep_mapping[entry]: + f.write(entry + ':' + dep + '\n') + + +def _ReadDepMapping(input_file_queue, output_dep_mapping_queue, + error_message_queue, output_root, file_open=open): + while True: + try: + input_file = input_file_queue.get_nowait() + except queue.Empty: + # No more work left in the queue. + return + + try: + deps = set() + input_file_name = os.path.splitext(input_file)[0] + entry = os.path.relpath(input_file_name, output_root) + for file_ext in ['.m', '.h']: + with file_open(input_file_name + file_ext, 'r') as f: + for line in f: + include = _INCLUDE_RE.match(line) + if include: + include_path = include.group(2) + dep = os.path.splitext(include_path)[0] + if dep != entry: + deps.add(dep) + + output_dep_mapping_queue.put((entry, sorted(deps))) + except Exception as e: # pylint: disable=broad-except + error_message_queue.put(str(e)) + finally: + # We need to mark the task done to prevent blocking the main process + # indefinitely. + input_file_queue.task_done() + + +def WriteArchiveSourceMappingFile(compiled_archive_file_path, + output_archive_source_mapping_file, + objc_files, + file_open=open): + """Writes a mapping file between archive file to associated ObjC source files. + + Args: + compiled_archive_file_path: The path of the archive file. + output_archive_source_mapping_file: A path of the mapping file to write to. + objc_files: A list of ObjC files translated by J2ObjC. + file_open: Reference to the builtin open function so it may be + overridden for testing. + Returns: + None. + """ + with file_open(output_archive_source_mapping_file, 'w') as f: + for objc_file in objc_files: + f.write(compiled_archive_file_path + ':' + objc_file + '\n') + + +def _ParseArgs(j2objc_args): + """Separate arguments passed to J2ObjC into source files and J2ObjC flags. + + Args: + j2objc_args: A list of args to pass to J2ObjC transpiler. + Returns: + A tuple containing source files and J2ObjC flags + """ + source_files = [] + flags = [] + is_next_flag_value = False + for j2objc_arg in j2objc_args: + if j2objc_arg.startswith('-'): + flags.append(j2objc_arg) + is_next_flag_value = True + elif is_next_flag_value: + flags.append(j2objc_arg) + is_next_flag_value = False + else: + source_files.append(j2objc_arg) + return (source_files, flags) + + +def _J2ObjcOutputObjcFiles(java_files): + """Returns the relative paths of the associated output ObjC source files. + + Args: + java_files: The list of Java files to translate. + Returns: + A list of associated output ObjC source files. + """ + return [os.path.splitext(java_file)[0] + '.m' for java_file in java_files] + + +def UnzipSourceJarSources(source_jars): + """Unzips the source jars containing Java source files. + + Args: + source_jars: The list of input Java source jars. + Returns: + A tuple of the temporary output root and a list of root-relative paths of + unzipped Java files + """ + srcjar_java_files = [] + if source_jars: + tmp_input_root = tempfile.mkdtemp() + for source_jar in source_jars: + zip_ref = zipfile.ZipFile(source_jar, 'r') + zip_entries = [] + + for file_entry in zip_ref.namelist(): + # We only care about Java source files. + if file_entry.endswith('.java'): + zip_entries.append(file_entry) + + zip_ref.extractall(tmp_input_root, zip_entries) + zip_ref.close() + srcjar_java_files.extend(zip_entries) + + return (tmp_input_root, srcjar_java_files) + else: + return None + + +def RenameGenJarObjcFileRootInFileContent(tmp_objc_file_root, + j2objc_source_paths, + gen_src_jar, genjar_objc_files, + execute=subprocess.check_call): + """Renames references to temporary root inside ObjC sources from gen srcjar. + + Args: + tmp_objc_file_root: The temporary output root containing ObjC sources. + j2objc_source_paths: The source paths used by J2ObjC. + gen_src_jar: The path of the gen srcjar. + genjar_objc_files: The list of ObjC sources translated from the gen srcjar. + execute: The function used to execute shell commands. + Returns: + None. + """ + if genjar_objc_files: + abs_genjar_objc_source_files = [ + os.path.join(tmp_objc_file_root, genjar_objc_f) + for genjar_objc_f in genjar_objc_files + ] + abs_genjar_objc_header_files = [ + os.path.join(tmp_objc_file_root, + os.path.splitext(genjar_objc_f)[0] + '.h') + for genjar_objc_f in genjar_objc_files + ] + + # We execute a command to change all references of the temporary Java root + # where we unzipped the gen srcjar sources, to the actual gen srcjar that + # contains the original Java sources. + cmd = [ + 'sed', + '-i', + '-e', + 's|%s/|%s::|g' % (j2objc_source_paths[1], gen_src_jar) + ] + cmd.extend(abs_genjar_objc_source_files) + cmd.extend(abs_genjar_objc_header_files) + execute(cmd, stderr=subprocess.STDOUT) + + +def MoveObjcFileToFinalOutputRoot(objc_files, + tmp_objc_file_root, + final_objc_file_root, + suffix, + os_module=os, + shutil_module=shutil): + """Moves ObjC files from temporary location to the final output location. + + Args: + objc_files: The list of objc files to move. + tmp_objc_file_root: The temporary output root containing ObjC sources. + final_objc_file_root: The final output root. + suffix: The suffix of the files to move. + os_module: The os python module. + shutil_module: The shutil python module. + Returns: + None. + """ + for objc_file in objc_files: + file_with_suffix = os_module.path.splitext(objc_file)[0] + suffix + dest_path = os_module.path.join( + final_objc_file_root, file_with_suffix) + dest_path_dir = os_module.path.dirname(dest_path) + + if not os_module.path.isdir(dest_path_dir): + try: + os_module.makedirs(dest_path_dir) + except OSError as e: + if e.errno != errno.EEXIST or not os_module.path.isdir(dest_path_dir): + raise + + shutil_module.move( + os_module.path.join(tmp_objc_file_root, file_with_suffix), + dest_path) + + +def PostJ2ObjcFileProcessing(normal_objc_files, genjar_objc_files, + tmp_objc_file_root, final_objc_file_root, + j2objc_source_paths, gen_src_jar, + output_gen_source_dir, output_gen_header_dir): + """Performs cleanups on ObjC files and moves them to final output location. + + Args: + normal_objc_files: The list of objc files translated from normal Java files. + genjar_objc_files: The list of ObjC sources translated from the gen srcjar. + tmp_objc_file_root: The temporary output root containing ObjC sources. + final_objc_file_root: The final output root. + j2objc_source_paths: The source paths used by J2ObjC. + gen_src_jar: The path of the gen srcjar. + output_gen_source_dir: The final output directory of ObjC source files + translated from gen srcjar. Maybe null. + output_gen_header_dir: The final output directory of ObjC header files + translated from gen srcjar. Maybe null. + Returns: + None. + """ + RenameGenJarObjcFileRootInFileContent(tmp_objc_file_root, + j2objc_source_paths, + gen_src_jar, + genjar_objc_files) + MoveObjcFileToFinalOutputRoot(normal_objc_files, + tmp_objc_file_root, + final_objc_file_root, + '.m') + MoveObjcFileToFinalOutputRoot(normal_objc_files, + tmp_objc_file_root, + final_objc_file_root, + '.h') + + if output_gen_source_dir: + MoveObjcFileToFinalOutputRoot( + genjar_objc_files, + tmp_objc_file_root, + output_gen_source_dir, + '.m') + + if output_gen_header_dir: + MoveObjcFileToFinalOutputRoot( + genjar_objc_files, + tmp_objc_file_root, + output_gen_header_dir, + '.h') + + +def GenerateJ2objcMappingFiles(normal_objc_files, + genjar_objc_files, + tmp_objc_file_root, + output_dependency_mapping_file, + output_archive_source_mapping_file, + compiled_archive_file_path): + """Generates J2ObjC mapping files. + + Args: + normal_objc_files: The list of objc files translated from normal Java files. + genjar_objc_files: The list of ObjC sources translated from the gen srcjar. + tmp_objc_file_root: The temporary output root containing ObjC sources. + output_dependency_mapping_file: The path of the dependency mapping file to + write to. + output_archive_source_mapping_file: A path of the mapping file to write to. + compiled_archive_file_path: The path of the archive file. + Returns: + None. + """ + WriteDepMappingFile(normal_objc_files + genjar_objc_files, + tmp_objc_file_root, + output_dependency_mapping_file) + + if output_archive_source_mapping_file: + WriteArchiveSourceMappingFile(compiled_archive_file_path, + output_archive_source_mapping_file, + normal_objc_files + genjar_objc_files) + + +def main(): + parser = argparse.ArgumentParser(fromfile_prefix_chars='@') + parser.add_argument( + '--java', + required=True, + help='The path to the Java executable.') + parser.add_argument( + '--jvm_flags', + default='-Xss4m,-XX:+UseParallelGC', + help='A comma-separated list of flags to pass to the JVM.') + parser.add_argument( + '--j2objc', + required=True, + help='The path to the J2ObjC deploy jar.') + parser.add_argument( + '--main_class', + required=True, + help='The main class of the J2ObjC deploy jar to execute.') + # TODO(rduan): Remove, no longer needed. + parser.add_argument( + '--translated_source_files', + required=False, + help=('A comma-separated list of file paths where J2ObjC will write the ' + 'translated files to.')) + parser.add_argument( + '--output_dependency_mapping_file', + required=True, + help='The file path of the dependency mapping file to write to.') + parser.add_argument( + '--objc_file_path', '-d', + required=True, + help=('The file path which represents a directory where the generated ' + 'ObjC files reside.')) + parser.add_argument( + '--output_archive_source_mapping_file', + help='The file path of the mapping file containing mappings between the ' + 'translated source files and the to-be-generated archive file ' + 'compiled from those source files. --compile_archive_file_path must ' + 'be specified if this option is specified.') + parser.add_argument( + '--compiled_archive_file_path', + required=False, + help=('The archive file path that will be produced by ObjC compile action' + ' later')) + # TODO(rduan): Remove this flag once it is fully replaced by flag --src_jars. + parser.add_argument( + '--gen_src_jar', + required=False, + help='The jar containing Java sources generated by annotation processor.') + parser.add_argument( + '--src_jars', + required=False, + help='The list of Java source jars containing Java sources to translate.') + parser.add_argument( + '--output_gen_source_dir', + required=False, + help='The output directory of ObjC source files translated from the gen' + ' srcjar') + parser.add_argument( + '--output_gen_header_dir', + required=False, + help='The output directory of ObjC header files translated from the gen' + ' srcjar') + + args, pass_through_args = parser.parse_known_args() + normal_java_files, j2objc_flags = _ParseArgs(pass_through_args) + srcjar_java_files = [] + j2objc_source_paths = [os.getcwd()] + + # Unzip the source jars, so J2ObjC can translate the contained sources. + # Also add the temporary directory containing the unzipped sources as a source + # path for J2ObjC, so it can find these sources. + source_jars = [] + if args.gen_src_jar: + source_jars.append(args.gen_src_jar) + if args.src_jars: + source_jars.extend(args.src_jars.split(',')) + + srcjar_source_tuple = UnzipSourceJarSources(source_jars) + if srcjar_source_tuple: + j2objc_source_paths.append(srcjar_source_tuple[0]) + srcjar_java_files = srcjar_source_tuple[1] + + # Run J2ObjC over the normal input Java files and unzipped gen jar Java files. + # The output is stored in a temporary directory. + tmp_objc_file_root = tempfile.mkdtemp() + + # If we do not generate the header mapping from J2ObjC, we still + # need to specify --output-header-mapping, as it signals to J2ObjC that we + # are using source paths as import paths, not package paths. + # TODO(rduan): Make another flag in J2ObjC to specify using source paths. + if '--output-header-mapping' not in j2objc_flags: + j2objc_flags.extend(['--output-header-mapping', '/dev/null']) + + RunJ2ObjC(args.java, + args.jvm_flags, + args.j2objc, + args.main_class, + tmp_objc_file_root, + j2objc_flags, + j2objc_source_paths, + normal_java_files + srcjar_java_files) + + # Calculate the relative paths of generated objc files. + normal_objc_files = _J2ObjcOutputObjcFiles(normal_java_files) + genjar_objc_files = _J2ObjcOutputObjcFiles(srcjar_java_files) + + # Generate J2ObjC mapping files needed for distributed builds. + GenerateJ2objcMappingFiles(normal_objc_files, + genjar_objc_files, + tmp_objc_file_root, + args.output_dependency_mapping_file, + args.output_archive_source_mapping_file, + args.compiled_archive_file_path) + + # Post J2ObjC-run processing, involving file editing, zipping and moving + # files to their final output locations. + PostJ2ObjcFileProcessing( + normal_objc_files, + genjar_objc_files, + tmp_objc_file_root, + args.objc_file_path, + j2objc_source_paths, + args.gen_src_jar, + args.output_gen_source_dir, + args.output_gen_header_dir) + +if __name__ == '__main__': + main() diff --git a/tools/objc/BUILD b/tools/objc/BUILD index 32557d4a3ca209..b50136dce5edcc 100644 --- a/tools/objc/BUILD +++ b/tools/objc/BUILD @@ -32,6 +32,11 @@ filegroup( srcs = ["j2objc_dead_code_pruner.py"], ) +py_binary( + name = "j2objc_dead_code_pruner_binary", + srcs = ["j2objc_dead_code_pruner_binary.py"], +) + objc_library( name = "dummy_lib", srcs = [ diff --git a/tools/objc/j2objc_dead_code_pruner_binary.py b/tools/objc/j2objc_dead_code_pruner_binary.py new file mode 100755 index 00000000000000..005241281f8a4e --- /dev/null +++ b/tools/objc/j2objc_dead_code_pruner_binary.py @@ -0,0 +1,499 @@ +#!/usr/bin/python3 + +# Copyright 2015 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A script for J2ObjC dead code removal in Blaze. + +This script removes unused J2ObjC-translated classes from compilation and +linking by: + 1. Build a class dependency tree among translated source files. + 2. Use user-provided Java class entry points to get a list of reachable + classes. + 3. Go through all translated source files and rewrite unreachable ones with + dummy content. +""" + +import argparse +import collections +import multiprocessing +import os +import queue +import re +import shlex +import shutil +import subprocess +import threading + + +PRUNED_SRC_CONTENT = 'static int DUMMY_unused __attribute__((unused,used)) = 0;' + + +def BuildReachabilityTree(dependency_mapping_files, file_open=open): + """Builds a reachability tree using entries from dependency mapping files. + + Args: + dependency_mapping_files: A comma separated list of J2ObjC-generated + dependency mapping files. + file_open: Reference to the builtin open function so it may be + overridden for testing. + Returns: + A dict mapping J2ObjC-generated source files to the corresponding direct + dependent source files. + """ + return BuildArtifactSourceTree(dependency_mapping_files, file_open) + + +def BuildHeaderMapping(header_mapping_files, file_open=open): + """Builds a mapping between Java classes and J2ObjC-translated header files. + + Args: + header_mapping_files: A comma separated list of J2ObjC-generated + header mapping files. + file_open: Reference to the builtin open function so it may be + overridden for testing. + Returns: + An ordered dict mapping Java class names to corresponding J2ObjC-translated + source files. + """ + header_mapping = collections.OrderedDict() + for header_mapping_file in header_mapping_files.split(','): + with file_open(header_mapping_file, 'r') as f: + for line in f: + java_class_name = line.strip().split('=')[0] + transpiled_file_name = os.path.splitext(line.strip().split('=')[1])[0] + header_mapping[java_class_name] = transpiled_file_name + return header_mapping + + +def BuildReachableFileSet(entry_classes, reachability_tree, header_mapping, + archive_source_file_mapping=None): + """Builds a set of reachable translated files from entry Java classes. + + Args: + entry_classes: A comma separated list of Java entry classes. + reachability_tree: A dict mapping translated files to their direct + dependencies. + header_mapping: A dict mapping Java class names to translated source files. + archive_source_file_mapping: A dict mapping source files to the associated + archive file that contains them. + Returns: + A set of reachable translated files from the given list of entry classes. + Raises: + Exception: If there is an entry class that is not being transpiled in this + j2objc_library. + """ + transpiled_entry_files = [] + for entry_class in entry_classes.split(','): + if entry_class not in header_mapping: + raise Exception( + entry_class + + ' is not in the transitive Java deps of included ' + + 'j2objc_library rules.') + transpiled_entry_files.append(header_mapping[entry_class]) + + # Translated files going into the same static library archive with duplicated + # base names also need to be added to the set of entry files. + # + # This edge case is ignored because we currently cannot correctly perform + # dead code removal in this case. The object file entries in static library + # archives are named by the base names of the original source files. If two + # source files (e.g., foo/bar.m, bar/bar.m) go into the same archive and + # share the same base name (bar.m), their object file entries inside the + # archive will have the same name (bar.o). We cannot correctly handle this + # case because current archive tools (ar, ranlib, etc.) do not handle this + # case very well. + if archive_source_file_mapping: + transpiled_entry_files.extend(_DuplicatedFiles(archive_source_file_mapping)) + + # Translated files from package-info.java are also added to the entry files + # because they are needed to resolve ObjC class names with prefixes and these + # files may also have dependencies. + for transpiled_file in reachability_tree: + if transpiled_file.endswith('package-info'): + transpiled_entry_files.append(transpiled_file) + + reachable_files = set() + for transpiled_entry_file in transpiled_entry_files: + reachable_files.add(transpiled_entry_file) + current_level_deps = [] + # We need to check if the transpiled file is in the reachability tree + # because J2ObjC protos are not analyzed for dead code stripping and + # therefore are not in the reachability tree at all. + if transpiled_entry_file in reachability_tree: + current_level_deps = reachability_tree[transpiled_entry_file] + while current_level_deps: + next_level_deps = [] + for dep in current_level_deps: + if dep not in reachable_files: + reachable_files.add(dep) + if dep in reachability_tree: + next_level_deps.extend(reachability_tree[dep]) + current_level_deps = next_level_deps + return reachable_files + + +def PruneFiles(input_files, output_files, objc_file_path, reachable_files, + file_open=open, file_shutil=shutil): + """Copies over translated files and remove the contents of unreachable files. + + Args: + input_files: A comma separated list of input source files to prune. It has + a one-on-one pair mapping with the output_file list. + output_files: A comma separated list of output source files to write pruned + source files to. It has a one-on-one pair mapping with the input_file + list. + objc_file_path: The file path which represents a directory where the + generated ObjC files reside. + reachable_files: A set of reachable source files. + file_open: Reference to the builtin open function so it may be + overridden for testing. + file_shutil: Reference to the builtin shutil module so it may be + overridden for testing. + Returns: + None. + """ + file_queue = queue.queue() + for input_file, output_file in zip( + input_files.split(','), + output_files.split(',')): + file_queue.put((input_file, output_file)) + + for _ in range(multiprocessing.cpu_count()): + t = threading.Thread(target=_PruneFile, args=(file_queue, + reachable_files, + objc_file_path, + file_open, + file_shutil)) + t.start() + + file_queue.join() + + +def _PruneFile(file_queue, reachable_files, objc_file_path, file_open=open, + file_shutil=shutil): + while True: + try: + input_file, output_file = file_queue.get_nowait() + except queue.Empty: + return + file_name = os.path.relpath(os.path.splitext(input_file)[0], + objc_file_path) + if file_name in reachable_files: + file_shutil.copy(input_file, output_file) + else: + with file_open(output_file, 'w') as f: + # Use a static variable scoped to the source file to suppress + # the "has no symbols" linker warning for empty object files. + f.write(PRUNED_SRC_CONTENT) + file_queue.task_done() + + +def _DuplicatedFiles(archive_source_file_mapping): + """Returns a list of file with duplicated base names in each archive file. + + Args: + archive_source_file_mapping: A dict mapping source files to the associated + archive file that contains them. + Returns: + A list containing files with duplicated base names. + """ + duplicated_files = [] + dict_with_duplicates = dict() + + for source_files in archive_source_file_mapping.values(): + for source_file in source_files: + file_basename = os.path.basename(source_file) + file_without_ext = os.path.splitext(source_file)[0] + if file_basename in dict_with_duplicates: + dict_with_duplicates[file_basename].append(file_without_ext) + else: + dict_with_duplicates[file_basename] = [file_without_ext] + for basename in dict_with_duplicates: + if len(dict_with_duplicates[basename]) > 1: + duplicated_files.extend(dict_with_duplicates[basename]) + dict_with_duplicates = dict() + + return duplicated_files + + +def BuildArchiveSourceFileMapping(archive_source_mapping_files, file_open): + """Builds a mapping between archive files and their associated source files. + + Args: + archive_source_mapping_files: A comma separated list of J2ObjC-generated + mapping between archive files and their associated source files. + file_open: Reference to the builtin open function so it may be + overridden for testing. + Returns: + A dict mapping between archive files and their associated source files. + """ + return BuildArtifactSourceTree(archive_source_mapping_files, file_open) + + +def PruneSourceFiles(input_files, output_files, dependency_mapping_files, + header_mapping_files, entry_classes, objc_file_path, + file_open=open, file_shutil=shutil): + """Copies over translated files and remove the contents of unreachable files. + + Args: + input_files: A comma separated list of input source files to prune. It has + a one-on-one pair mapping with the output_file list. + output_files: A comma separated list of output source files to write pruned + source files to. It has a one-on-one pair mapping with the input_file + list. + dependency_mapping_files: A comma separated list of J2ObjC-generated + dependency mapping files. + header_mapping_files: A comma separated list of J2ObjC-generated + header mapping files. + entry_classes: A comma separated list of Java entry classes. + objc_file_path: The file path which represents a directory where the + generated ObjC files reside. + file_open: Reference to the builtin open function so it may be + overridden for testing. + file_shutil: Reference to the builtin shutil module so it may be + overridden for testing. + """ + reachability_file_mapping = BuildReachabilityTree( + dependency_mapping_files, file_open) + header_map = BuildHeaderMapping(header_mapping_files, file_open) + reachable_files_set = BuildReachableFileSet(entry_classes, + reachability_file_mapping, + header_map) + PruneFiles(input_files, + output_files, + objc_file_path, + reachable_files_set, + file_open, + file_shutil) + + +def MatchObjectNamesInArchive(xcrunwrapper, archive, object_names): + """Returns object names matching their identity in an archive file. + + The linker that blaze uses appends an md5 hash to object file + names prior to inclusion in the archive file. Thus, object names + such as 'foo.o' need to be matched to their appropriate name in + the archive file, such as 'foo_.o'. + + Args: + xcrunwrapper: A wrapper script over xcrun. + archive: The location of the archive file. + object_names: The expected basenames of object files to match, + sans extension. For example 'foo' (not 'foo.o'). + Returns: + A list of basenames of matching members of the given archive + """ + ar_contents_cmd = [xcrunwrapper, 'ar', '-t', archive] + real_object_names_output = subprocess.check_output(ar_contents_cmd) + real_object_names = real_object_names_output.decode('utf-8') + expected_object_name_regex = r'^(?:%s)(?:_[0-9a-f]{32}(?:-[0-9]+)?)?\.o$' % ( + '|'.join([re.escape(name) for name in object_names])) + return re.findall( + expected_object_name_regex, + real_object_names, + flags=re.MULTILINE) + + +def PruneArchiveFile(input_archive, output_archive, dummy_archive, + dependency_mapping_files, header_mapping_files, + archive_source_mapping_files, entry_classes, xcrunwrapper, + file_open=open): + """Remove unreachable objects from archive file. + + Args: + input_archive: The source archive file to prune. + output_archive: The location of the pruned archive file. + dummy_archive: A dummy archive file that contains no object. + dependency_mapping_files: A comma separated list of J2ObjC-generated + dependency mapping files. + header_mapping_files: A comma separated list of J2ObjC-generated + header mapping files. + archive_source_mapping_files: A comma separated list of J2ObjC-generated + mapping between archive files and their associated source files. + entry_classes: A comma separated list of Java entry classes. + xcrunwrapper: A wrapper script over xcrun. + file_open: Reference to the builtin open function so it may be + overridden for testing. + """ + reachability_file_mapping = BuildReachabilityTree( + dependency_mapping_files, file_open) + header_map = BuildHeaderMapping(header_mapping_files, file_open) + archive_source_file_mapping = BuildArchiveSourceFileMapping( + archive_source_mapping_files, file_open) + reachable_files_set = BuildReachableFileSet(entry_classes, + reachability_file_mapping, + header_map, + archive_source_file_mapping) + + # Copy the current processes' environment, as xcrunwrapper depends on these + # variables. + cmd_env = dict(os.environ) + j2objc_cmd = '' + if input_archive in archive_source_file_mapping: + source_files = archive_source_file_mapping[input_archive] + unreachable_object_names = [] + + for source_file in source_files: + if os.path.splitext(source_file)[0] not in reachable_files_set: + unreachable_object_names.append( + os.path.basename(os.path.splitext(source_file)[0])) + + # There are unreachable objects in the archive to prune + if unreachable_object_names: + # If all objects in the archive are unreachable, just copy over a dummy + # archive that contains no object + if len(unreachable_object_names) == len(source_files): + j2objc_cmd = 'cp %s %s' % (shlex.quote(dummy_archive), + shlex.quote(output_archive)) + # Else we need to prune the archive of unreachable objects + else: + cmd_env['ZERO_AR_DATE'] = '1' + # Copy the input archive to the output location + j2objc_cmd += 'cp %s %s && ' % (shlex.quote(input_archive), + shlex.quote(output_archive)) + # Make the output archive editable + j2objc_cmd += 'chmod +w %s && ' % (shlex.quote(output_archive)) + # Remove the unreachable objects from the archive + unreachable_object_names = MatchObjectNamesInArchive( + xcrunwrapper, input_archive, unreachable_object_names) + j2objc_cmd += '%s ar -d -s %s %s && ' % ( + shlex.quote(xcrunwrapper), + shlex.quote(output_archive), + ' '.join(shlex.quote(uon) for uon in unreachable_object_names)) + # Update the table of content of the archive file + j2objc_cmd += '%s ranlib %s' % (shlex.quote(xcrunwrapper), + shlex.quote(output_archive)) + # There are no unreachable objects, we just copy over the original archive + else: + j2objc_cmd = 'cp %s %s' % (shlex.quote(input_archive), + shlex.quote(output_archive)) + # The archive cannot be pruned by J2ObjC dead code removal, just copy over + # the original archive + else: + j2objc_cmd = 'cp %s %s' % (shlex.quote(input_archive), + shlex.quote(output_archive)) + + try: + subprocess.check_output( + j2objc_cmd, stderr=subprocess.STDOUT, shell=True, env=cmd_env) + except OSError as e: + raise Exception( + 'executing command failed: %s (%s)' % (j2objc_cmd, e.strerror)) + + # "Touch" the output file. + # Prevents a pre-Xcode-8 bug in which passing zero-date archive files to ld + # would cause ld to error. + os.utime(output_archive, None) + + +def BuildArtifactSourceTree(files, file_open=open): + """Builds a dependency tree using from dependency mapping files. + + Args: + files: A comma separated list of dependency mapping files. + file_open: Reference to the builtin open function so it may be overridden for + testing. + + Returns: + A dict mapping build artifacts (possibly generated source files) to the + corresponding direct dependent source files. + """ + tree = dict() + if not files: + return tree + for filename in files.split(','): + with file_open(filename, 'r') as f: + for line in f: + entry = line.strip().split(':')[0] + dep = line.strip().split(':')[1] + if entry in tree: + tree[entry].append(dep) + else: + tree[entry] = [dep] + return tree + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(fromfile_prefix_chars='@') + + # TODO(rduan): Remove these three flags once J2ObjC compile actions are fully + # moved to the edges. + parser.add_argument( + '--input_files', + help=('The comma-separated file paths of translated source files to ' + 'prune.')) + parser.add_argument( + '--output_files', + help='The comma-separated file paths of pruned source files to write to.') + parser.add_argument( + '--objc_file_path', + help='The file path which represents a directory where the generated ObjC' + ' files reside') + + parser.add_argument( + '--input_archive', + help=('The path of the translated archive to prune.')) + parser.add_argument( + '--output_archive', + help='The path of the pruned archive file to write to.') + parser.add_argument( + '--dummy_archive', + help='The dummy archive file that contains no symbol.') + parser.add_argument( + '--dependency_mapping_files', + help='The comma-separated file paths of dependency mapping files.') + parser.add_argument( + '--header_mapping_files', + help='The comma-separated file paths of header mapping files.') + parser.add_argument( + '--archive_source_mapping_files', + help='The comma-separated file paths of archive to source mapping files.' + 'These mapping files should contain mappings between the ' + 'translated source files and the archive file compiled from those ' + 'source files.') + parser.add_argument( + '--entry_classes', + help=('The comma-separated list of Java entry classes to be used as entry' + ' point of the dead code analysis.')) + parser.add_argument( + '--xcrunwrapper', + help=('The xcrun wrapper script.')) + + args = parser.parse_args() + + if not args.entry_classes: + raise Exception('J2objC dead code removal is on but no entry class is ', + 'specified in any j2objc_library targets in the transitive', + ' closure') + if args.input_archive and args.output_archive: + PruneArchiveFile( + args.input_archive, + args.output_archive, + args.dummy_archive, + args.dependency_mapping_files, + args.header_mapping_files, + args.archive_source_mapping_files, + args.entry_classes, + args.xcrunwrapper) + else: + # TODO(rduan): Remove once J2ObjC compile actions are fully moved to the + # edges. + PruneSourceFiles( + args.input_files, + args.output_files, + args.dependency_mapping_files, + args.header_mapping_files, + args.entry_classes, + args.objc_file_path)