fix(smali-input): compile one smali file at a time to avoid 64k limit (…

…#2158)

jadx-core/src/main/java/jadx/api/plugins/options/impl/BasePluginOptionsBuilder.java

@@ -64,6 +64,14 @@ public OptionBuilder<String> strOption(String name) {
 						.parser(v -> v));
 	}
 
+	public OptionBuilder<Integer> intOption(String name) {
+		return addOption(
+				new OptionData<Integer>(name)
+						.type(OptionType.NUMBER)
+						.formatter(Object::toString)
+						.parser(Integer::parseInt));
+	}
+
 	public <E extends Enum<?>> OptionBuilder<E> enumOption(String name, E[] values, Function<String, E> valueOf) {
 		return addOption(
 				new OptionData<E>(name)

jadx-plugins/jadx-dex-input/src/main/java/jadx/plugins/input/dex/DexInputPlugin.java

@@ -5,6 +5,7 @@
 import java.nio.file.Path;
 import java.util.Collections;
 import java.util.List;
+import java.util.stream.Collectors;
 
 import org.jetbrains.annotations.Nullable;
 
@@ -14,6 +15,7 @@
 import jadx.api.plugins.input.ICodeLoader;
 import jadx.api.plugins.input.data.impl.EmptyCodeLoader;
 import jadx.api.plugins.utils.CommonFileUtils;
+import jadx.plugins.input.dex.utils.IDexData;
 
 public class DexInputPlugin implements JadxPlugin {
 	public static final String PLUGIN_ID = "dex-input";
@@ -57,4 +59,11 @@ public ICodeLoader loadDexFromInputStream(InputStream in, @Nullable String fileL
 			throw new DexException("Failed to read input stream", e);
 		}
 	}
+
+	public ICodeLoader loadDexData(List<IDexData> list) {
+		List<DexReader> readers = list.stream()
+				.map(data -> loader.loadDexReader(data.getFileName(), data.getContent()))
+				.collect(Collectors.toList());
+		return new DexLoadResult(readers, null);
+	}
 }

jadx-plugins/jadx-dex-input/src/main/java/jadx/plugins/input/dex/utils/IDexData.java

@@ -0,0 +1,8 @@
+package jadx.plugins.input.dex.utils;
+
+public interface IDexData {
+
+	String getFileName();
+
+	byte[] getContent();
+}

jadx-plugins/jadx-dex-input/src/main/java/jadx/plugins/input/dex/utils/SimpleDexData.java

@@ -0,0 +1,28 @@
+package jadx.plugins.input.dex.utils;
+
+import java.util.Objects;
+
+public class SimpleDexData implements IDexData {
+	private final String fileName;
+	private final byte[] content;
+
+	public SimpleDexData(String fileName, byte[] content) {
+		this.fileName = Objects.requireNonNull(fileName);
+		this.content = Objects.requireNonNull(content);
+	}
+
+	@Override
+	public String getFileName() {
+		return fileName;
+	}
+
+	@Override
+	public byte[] getContent() {
+		return content;
+	}
+
+	@Override
+	public String toString() {
+		return "DexData{" + fileName + ", size=" + content.length + '}';
+	}
+}

jadx-plugins/jadx-smali-input/src/main/java/jadx/plugins/input/smali/SmaliConvert.java

@@ -1,79 +1,100 @@
 package jadx.plugins.input.smali;
 
 import java.io.ByteArrayOutputStream;
-import java.io.Closeable;
-import java.io.IOException;
 import java.io.OutputStream;
 import java.io.PrintStream;
+import java.io.Reader;
 import java.nio.file.FileSystems;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.PathMatcher;
-import java.util.Collections;
+import java.util.ArrayList;
 import java.util.List;
-import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
-import org.jetbrains.annotations.Nullable;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.android.tools.smali.smali.Smali;
 import com.android.tools.smali.smali.SmaliOptions;
 
-public class SmaliConvert implements Closeable {
+import jadx.plugins.input.dex.utils.IDexData;
+import jadx.plugins.input.dex.utils.SimpleDexData;
+
+public class SmaliConvert {
 	private static final Logger LOG = LoggerFactory.getLogger(SmaliConvert.class);
 
-	@Nullable
-	private Path tmpDex;
+	private final List<IDexData> dexData = new ArrayList<>();
 
-	public boolean execute(List<Path> input) {
+	public boolean execute(List<Path> input, SmaliInputOptions options) {
 		List<Path> smaliFiles = filterSmaliFiles(input);
 		if (smaliFiles.isEmpty()) {
 			return false;
 		}
-		LOG.debug("Compiling smali files: {}", smaliFiles.size());
-		try {
-			this.tmpDex = Files.createTempFile("jadx-", ".dex");
-			if (compileSmali(tmpDex, smaliFiles)) {
-				return true;
+		try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
+			collectSystemErrors(out, () -> compile(smaliFiles, options));
+			boolean success = out.size() == 0;
+			if (!success) {
+				LOG.error("Smali error:\n{}", out);
 			}
 		} catch (Exception e) {
 			LOG.error("Smali process error", e);
 		}
-		close();
-		return false;
+		return !dexData.isEmpty();
 	}
 
-	private static boolean compileSmali(Path output, List<Path> inputFiles) throws IOException {
-		SmaliOptions options = new SmaliOptions();
-		options.outputDexFile = output.toAbsolutePath().toString();
-		options.verboseErrors = true;
-		options.apiLevel = 27; // TODO: add as plugin option
+	@SuppressWarnings("ResultOfMethodCallIgnored")
+	private void compile(List<Path> inputFiles, SmaliInputOptions options) {
+		SmaliOptions smaliOptions = new SmaliOptions();
+		smaliOptions.apiLevel = options.getApiLevel();
+		smaliOptions.verboseErrors = true;
+		smaliOptions.allowOdexOpcodes = false;
+		smaliOptions.printTokens = false;
 
-		List<String> inputFileNames = inputFiles.stream()
-				.map(p -> p.toAbsolutePath().toString())
-				.distinct()
-				.collect(Collectors.toList());
-
-		try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
-			boolean result = collectSystemErrors(out, () -> Smali.assemble(options, inputFileNames));
-			if (!result) {
-				LOG.error("Smali compilation error:\n{}", out);
+		int threads = options.getThreads();
+		LOG.debug("Compiling smali files: {}, threads: {}", inputFiles.size(), threads);
+		long start = System.currentTimeMillis();
+		if (threads == 1) {
+			for (Path inputFile : inputFiles) {
+				assemble(inputFile, smaliOptions);
 			}
-			return result;
+		} else {
+			try {
+				ExecutorService executor = Executors.newFixedThreadPool(threads);
+				for (Path inputFile : inputFiles) {
+					executor.execute(() -> assemble(inputFile, smaliOptions));
+				}
+				executor.shutdown();
+				executor.awaitTermination(1, TimeUnit.DAYS);
+			} catch (InterruptedException e) {
+				LOG.error("Smali compile interrupted", e);
+			}
+		}
+		if (LOG.isDebugEnabled()) {
+			LOG.debug("Smali compile done in: {}ms", System.currentTimeMillis() - start);
+		}
+	}
+
+	private void assemble(Path inputFile, SmaliOptions smaliOptions) {
+		String fileName = inputFile.toAbsolutePath().toString();
+		try (Reader reader = Files.newBufferedReader(inputFile)) {
+			byte[] assemble = SmaliUtils.assemble(reader, smaliOptions);
+			dexData.add(new SimpleDexData(fileName, assemble));
+		} catch (Exception e) {
+			throw new RuntimeException("Fail to compile: " + fileName, e);
 		}
 	}
 
-	private static boolean collectSystemErrors(OutputStream out, Callable<Boolean> exec) {
+	private static void collectSystemErrors(OutputStream out, Runnable exec) {
 		PrintStream systemErr = System.err;
 		try (PrintStream err = new PrintStream(out)) {
 			System.setErr(err);
 			try {
-				return exec.call();
+				exec.run();
 			} catch (Exception e) {
 				e.printStackTrace(err);
-				return false;
 			}
 		} finally {
 			System.setErr(systemErr);
@@ -87,21 +108,7 @@ private List<Path> filterSmaliFiles(List<Path> input) {
 				.collect(Collectors.toList());
 	}
 
-	public List<Path> getDexFiles() {
-		if (tmpDex == null) {
-			return Collections.emptyList();
-		}
-		return Collections.singletonList(tmpDex);
-	}
-
-	@Override
-	public void close() {
-		try {
-			if (tmpDex != null) {
-				Files.deleteIfExists(tmpDex);
-			}
-		} catch (Exception e) {
-			LOG.error("Failed to remove tmp dex file: {}", tmpDex, e);
-		}
+	public List<IDexData> getDexData() {
+		return dexData;
 	}
 }

jadx-plugins/jadx-smali-input/src/main/java/jadx/plugins/input/smali/SmaliInputOptions.java

@@ -0,0 +1,29 @@
+package jadx.plugins.input.smali;
+
+import jadx.api.plugins.options.impl.BasePluginOptionsBuilder;
+
+public class SmaliInputOptions extends BasePluginOptionsBuilder {
+
+	private int apiLevel;
+	private int threads; // use jadx global threads count option
+
+	@Override
+	public void registerOptions() {
+		intOption(SmaliInputPlugin.PLUGIN_ID + ".api-level")
+				.description("Android API level")
+				.defaultValue(27)
+				.setter(v -> apiLevel = v);
+	}
+
+	public int getApiLevel() {
+		return apiLevel;
+	}
+
+	public int getThreads() {
+		return threads;
+	}
+
+	public void setThreads(int threads) {
+		this.threads = threads;
+	}
+}

jadx-plugins/jadx-smali-input/src/main/java/jadx/plugins/input/smali/SmaliInputPlugin.java

@@ -3,26 +3,31 @@
 import jadx.api.plugins.JadxPlugin;
 import jadx.api.plugins.JadxPluginContext;
 import jadx.api.plugins.JadxPluginInfo;
-import jadx.api.plugins.data.JadxPluginRuntimeData;
 import jadx.api.plugins.input.data.impl.EmptyCodeLoader;
 import jadx.plugins.input.dex.DexInputPlugin;
 
 public class SmaliInputPlugin implements JadxPlugin {
+	public static final String PLUGIN_ID = "smali-input";
+
+	private final SmaliInputOptions options = new SmaliInputOptions();
 
 	@Override
 	public JadxPluginInfo getPluginInfo() {
-		return new JadxPluginInfo("smali-input", "Smali Input", "Load .smali files");
+		return new JadxPluginInfo(PLUGIN_ID, "Smali Input", "Load .smali files");
 	}
 
 	@Override
 	public void init(JadxPluginContext context) {
-		JadxPluginRuntimeData dexInput = context.plugins().getById(DexInputPlugin.PLUGIN_ID);
+		context.registerOptions(options);
+		options.setThreads(context.getArgs().getThreadsCount());
+
+		DexInputPlugin dexInput = context.plugins().getInstance(DexInputPlugin.class);
 		context.addCodeInput(input -> {
 			SmaliConvert convert = new SmaliConvert();
-			if (!convert.execute(input)) {
+			if (!convert.execute(input, options)) {
 				return EmptyCodeLoader.INSTANCE;
 			}
-			return dexInput.loadCodeFiles(convert.getDexFiles(), convert);
+			return dexInput.loadDexData(convert.getDexData());
 		});
 	}
 }

jadx-plugins/jadx-smali-input/src/main/java/jadx/plugins/input/smali/SmaliUtils.java

@@ -0,0 +1,54 @@
+package jadx.plugins.input.smali;
+
+import java.io.IOException;
+import java.io.Reader;
+
+import org.antlr.runtime.CommonTokenStream;
+import org.antlr.runtime.RecognitionException;
+import org.antlr.runtime.TokenSource;
+import org.antlr.runtime.tree.CommonTreeNodeStream;
+
+import com.android.tools.smali.dexlib2.Opcodes;
+import com.android.tools.smali.dexlib2.writer.builder.DexBuilder;
+import com.android.tools.smali.dexlib2.writer.io.MemoryDataStore;
+import com.android.tools.smali.smali.LexerErrorInterface;
+import com.android.tools.smali.smali.SmaliOptions;
+import com.android.tools.smali.smali.smaliFlexLexer;
+import com.android.tools.smali.smali.smaliParser;
+import com.android.tools.smali.smali.smaliTreeWalker;
+
+/**
+ * Utility methods to assemble smali to in-memory buffer.
+ * This implementation uses smali library internal classes.
+ */
+public class SmaliUtils {
+
+	@SuppressWarnings("ExtractMethodRecommender")
+	public static byte[] assemble(Reader reader, SmaliOptions options) throws IOException, RecognitionException {
+		LexerErrorInterface lexer = new smaliFlexLexer(reader, options.apiLevel);
+		CommonTokenStream tokens = new CommonTokenStream((TokenSource) lexer);
+		smaliParser parser = new smaliParser(tokens);
+		parser.setVerboseErrors(options.verboseErrors);
+		parser.setAllowOdex(options.allowOdexOpcodes);
+		parser.setApiLevel(options.apiLevel);
+		smaliParser.smali_file_return parseResult = parser.smali_file();
+		if (parser.getNumberOfSyntaxErrors() > 0 || lexer.getNumberOfSyntaxErrors() > 0) {
+			throw new RuntimeException("Parse error");
+		}
+		CommonTreeNodeStream treeStream = new CommonTreeNodeStream(parseResult.getTree());
+		treeStream.setTokenStream(tokens);
+
+		DexBuilder dexBuilder = new DexBuilder(Opcodes.forApi(options.apiLevel));
+		smaliTreeWalker dexGen = new smaliTreeWalker(treeStream);
+		dexGen.setApiLevel(options.apiLevel);
+		dexGen.setVerboseErrors(options.verboseErrors);
+		dexGen.setDexBuilder(dexBuilder);
+		dexGen.smali_file();
+		if (dexGen.getNumberOfSyntaxErrors() > 0) {
+			throw new RuntimeException("Compile error");
+		}
+		MemoryDataStore dataStore = new MemoryDataStore();
+		dexBuilder.writeTo(dataStore);
+		return dataStore.getData();
+	}
+}