From e191b82f428e6323af0ece87809a6f1f816a0410 Mon Sep 17 00:00:00 2001 From: Matthieu Vergne Date: Sat, 20 Oct 2018 19:06:17 +0200 Subject: [PATCH 1/5] Add test for lambda retrieval. --- pom.xml | 18 ++++ src/test/java/javacg/JARBuilder.java | 117 ++++++++++++++++++++++ src/test/java/javacg/RunCucumberTest.java | 10 ++ src/test/java/javacg/StepDefinitions.java | 48 +++++++++ src/test/resources/javacg/lambda.feature | 37 +++++++ 5 files changed, 230 insertions(+) create mode 100644 src/test/java/javacg/JARBuilder.java create mode 100644 src/test/java/javacg/RunCucumberTest.java create mode 100644 src/test/java/javacg/StepDefinitions.java create mode 100644 src/test/resources/javacg/lambda.feature diff --git a/pom.xml b/pom.xml index 0992d00a..3b510209 100644 --- a/pom.xml +++ b/pom.xml @@ -26,6 +26,24 @@ 3.12.1.GA provided + + io.cucumber + cucumber-java + 2.3.1 + test + + + io.cucumber + cucumber-junit + 2.3.1 + test + + + junit + junit + 4.12 + test + diff --git a/src/test/java/javacg/JARBuilder.java b/src/test/java/javacg/JARBuilder.java new file mode 100644 index 00000000..2d63f615 --- /dev/null +++ b/src/test/java/javacg/JARBuilder.java @@ -0,0 +1,117 @@ +package javacg; + +import java.io.BufferedInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.StringWriter; +import java.net.URI; +import java.util.Arrays; +import java.util.Collection; +import java.util.LinkedList; +import java.util.jar.Attributes; +import java.util.jar.JarEntry; +import java.util.jar.JarOutputStream; +import java.util.jar.Manifest; + +import javax.tools.Diagnostic; +import javax.tools.DiagnosticCollector; +import javax.tools.JavaCompiler; +import javax.tools.JavaCompiler.CompilationTask; +import javax.tools.JavaFileObject; +import javax.tools.SimpleJavaFileObject; +import javax.tools.StandardJavaFileManager; +import javax.tools.StandardLocation; +import javax.tools.ToolProvider; + +public class JARBuilder { + private static final String TEMP_DIR = System.getProperty("java.io.tmpdir"); + + private final JavaCompiler compiler = ToolProvider.getSystemJavaCompiler(); + private final DiagnosticCollector diagnostics = new DiagnosticCollector(); + private final StandardJavaFileManager fileManager = compiler.getStandardFileManager(diagnostics, null, null); + private final LinkedList compilationUnits = new LinkedList<>(); + private final Collection classFiles = new LinkedList<>(); + + public JARBuilder() throws IOException { + fileManager.setLocation(StandardLocation.CLASS_OUTPUT, Arrays.asList(new File(TEMP_DIR))); + } + + public void add(String className, String classCode) throws IOException { + compilationUnits.add(createJavaFile(className, classCode)); + classFiles.add(new File(TEMP_DIR, className + ".class")); + } + + public File build() throws FileNotFoundException, IOException { + CompilationTask task = compiler.getTask(null, fileManager, diagnostics, null, null, compilationUnits); + boolean success = task.call(); + if (!success) { + displayDiagnostic(diagnostics); + throw new RuntimeException("Cannot compile classes for the JAR"); + } + + File file = File.createTempFile("test", ".jar"); + JarOutputStream jar = new JarOutputStream(new FileOutputStream(file), createManifest()); + for (File classFile : classFiles) { + add(classFile, jar); + } + jar.close(); + return file; + } + + private void displayDiagnostic(DiagnosticCollector diagnostics) { + for (Diagnostic diagnostic : diagnostics.getDiagnostics()) { + JavaSourceFromString sourceClass = (JavaSourceFromString) diagnostic.getSource(); + System.err.println("-----"); + System.err.println("Source: " + sourceClass.getName()); + System.err.println("Message: " + diagnostic.getMessage(null)); + System.err.println("Position: " + diagnostic.getPosition()); + System.err.println(diagnostic.getKind() + " " + diagnostic.getCode()); + } + } + + private JavaFileObject createJavaFile(String className, String classCode) throws IOException { + StringWriter writer = new StringWriter(); + writer.append(classCode); + writer.close(); + JavaFileObject file = new JavaSourceFromString(className, writer.toString()); + return file; + } + + private Manifest createManifest() { + Manifest manifest = new Manifest(); + manifest.getMainAttributes().put(Attributes.Name.MANIFEST_VERSION, "1.0"); + return manifest; + } + + private void add(File classFile, JarOutputStream jar) throws IOException { + JarEntry entry = new JarEntry(classFile.getPath().replace("\\", "/")); + jar.putNextEntry(entry); + try (BufferedInputStream in = new BufferedInputStream(new FileInputStream(classFile))) { + byte[] buffer = new byte[1024]; + while (true) { + int count = in.read(buffer); + if (count == -1) + break; + jar.write(buffer, 0, count); + } + jar.closeEntry(); + } + } +} + +class JavaSourceFromString extends SimpleJavaFileObject { + final String code; + + JavaSourceFromString(String name, String code) throws IOException { + super(URI.create("string:///" + name.replace('.', '/') + Kind.SOURCE.extension), Kind.SOURCE); + this.code = code; + } + + @Override + public CharSequence getCharContent(boolean ignoreEncodingErrors) { + return code; + } +} \ No newline at end of file diff --git a/src/test/java/javacg/RunCucumberTest.java b/src/test/java/javacg/RunCucumberTest.java new file mode 100644 index 00000000..9de478d1 --- /dev/null +++ b/src/test/java/javacg/RunCucumberTest.java @@ -0,0 +1,10 @@ +package javacg; + +import cucumber.api.CucumberOptions; +import cucumber.api.junit.Cucumber; +import org.junit.runner.RunWith; + +@RunWith(Cucumber.class) +@CucumberOptions(plugin = { "pretty" }) +public class RunCucumberTest { +} \ No newline at end of file diff --git a/src/test/java/javacg/StepDefinitions.java b/src/test/java/javacg/StepDefinitions.java new file mode 100644 index 00000000..05680698 --- /dev/null +++ b/src/test/java/javacg/StepDefinitions.java @@ -0,0 +1,48 @@ +package javacg; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.PrintStream; + +import cucumber.api.java.en.Given; +import cucumber.api.java.en.Then; +import cucumber.api.java.en.When; +import gr.gousiosg.javacg.stat.JCallGraph; + +public class StepDefinitions { + private final JARBuilder jarBuilder; + private String result; + + public StepDefinitions() throws IOException { + jarBuilder = new JARBuilder(); + } + + @Given("^I have the class \"([^\"]*)\" with code:$") + public void i_have_the_class_with_code(String className, String classCode) throws Exception { + jarBuilder.add(className, classCode); + } + + @When("^I run the analyze$") + public void i_analyze_it() throws Exception { + File jarFile = jarBuilder.build(); + + PrintStream oldOut = System.out; + ByteArrayOutputStream resultBuffer = new ByteArrayOutputStream(); + System.setOut(new PrintStream(resultBuffer)); + JCallGraph.main(new String[] { jarFile.getPath() }); + System.setOut(oldOut); + + result = resultBuffer.toString(); + } + + @Then("^the result should contain:$") + public void the_result_should_contain(String line) throws Exception { + if (result.contains(line)) { + // OK + } else { + System.err.println(result); + throw new RuntimeException("Cannot found: " + line); + } + } +} diff --git a/src/test/resources/javacg/lambda.feature b/src/test/resources/javacg/lambda.feature new file mode 100644 index 00000000..14d8f3fb --- /dev/null +++ b/src/test/resources/javacg/lambda.feature @@ -0,0 +1,37 @@ +#Author: matthieu.vergne@gmail.com +Feature: Lambda + I want to identify all lambdas within the analyzed code. + + Background: + # Introduce the lambda we will use + Given I have the class "Runner" with code: + """ + @FunctionalInterface + public interface Runner { + public void run(); + } + """ + + Scenario: Retrieve lambda in method + Given I have the class "LambdaTest" with code: + """ + public class LambdaTest { + public void methodA() { + Runner r = () -> methodB(); + r.run(); + } + + public void methodB() {} + } + """ + When I run the analyze + # Creation of r in methodA + Then the result should contain: + """ + M:LambdaTest:methodA() (D)Runner:run(LambdaTest) + """ + # Call of methodB in r + And the result should contain: + """ + M:LambdaTest:lambda$methodA$0() (M)LambdaTest:methodB() + """ From 6c06e512ed349b7d86c7c7df3683da991371b035 Mon Sep 17 00:00:00 2001 From: Matthieu Vergne Date: Sun, 21 Oct 2018 18:35:05 +0200 Subject: [PATCH 2/5] Highlight the broken call graphs of nested lambdas with a test. --- src/test/resources/javacg/lambda.feature | 40 ++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/src/test/resources/javacg/lambda.feature b/src/test/resources/javacg/lambda.feature index 14d8f3fb..c651e953 100644 --- a/src/test/resources/javacg/lambda.feature +++ b/src/test/resources/javacg/lambda.feature @@ -35,3 +35,43 @@ Feature: Lambda """ M:LambdaTest:lambda$methodA$0() (M)LambdaTest:methodB() """ + + Scenario: Retrieve nested lambdas + Given I have the class "NestedLambdaTest" with code: + """ + public class NestedLambdaTest { + public void methodA() { + Runner r = () -> { + Runner r2 = () -> { + Runner r3 = () -> methodB(); + r3.run(); + }; + r2.run(); + }; + r.run(); + } + + public void methodB() {} + } + """ + When I run the analyze + # Creation of r in methodA + Then the result should contain: + """ + M:NestedLambdaTest:methodA() (D)Runner:run(NestedLambdaTest) + """ + # Creation of r2 in r + And the result should contain: + """ + M:NestedLambdaTest:lambda$methodA$2() (D)Runner:run(NestedLambdaTest) + """ + # Creation of r3 in r2 + And the result should contain: + """ + M:NestedLambdaTest:lambda$lambda$methodA$2$1() (D)Runner:run(NestedLambdaTest) + """ + # Call of methodB in r3 + And the result should contain: + """ + M:NestedLambdaTest:lambda$lambda$lambda$methodA$2$1$0() (M)NestedLambdaTest:methodB() + """ From 324960093b229db121d5bae0a7bb9810f1643f15 Mon Sep 17 00:00:00 2001 From: Matthieu Vergne Date: Sun, 21 Oct 2018 19:45:36 +0200 Subject: [PATCH 3/5] Fix: Missing method name in nested lambdas (broken call graph). --- .../gr/gousiosg/javacg/stat/ClassVisitor.java | 10 +- .../javacg/stat/DynamicCallManager.java | 116 ++++++++++++++++++ 2 files changed, 124 insertions(+), 2 deletions(-) create mode 100644 src/main/java/gr/gousiosg/javacg/stat/DynamicCallManager.java diff --git a/src/main/java/gr/gousiosg/javacg/stat/ClassVisitor.java b/src/main/java/gr/gousiosg/javacg/stat/ClassVisitor.java index e03d72c6..d5dad604 100644 --- a/src/main/java/gr/gousiosg/javacg/stat/ClassVisitor.java +++ b/src/main/java/gr/gousiosg/javacg/stat/ClassVisitor.java @@ -45,6 +45,7 @@ public class ClassVisitor extends EmptyVisitor { private JavaClass clazz; private ConstantPoolGen constants; private String classReferenceFormat; + private final DynamicCallManager DCManager = new DynamicCallManager(); public ClassVisitor(JavaClass jc) { clazz = jc; @@ -55,8 +56,13 @@ public ClassVisitor(JavaClass jc) { public void visitJavaClass(JavaClass jc) { jc.getConstantPool().accept(this); Method[] methods = jc.getMethods(); - for (int i = 0; i < methods.length; i++) - methods[i].accept(this); + for (int i = 0; i < methods.length; i++) { + Method method = methods[i]; + DCManager.retrieveCalls(method, jc); + DCManager.linkCalls(method); + method.accept(this); + + } } public void visitConstantPool(ConstantPool constantPool) { diff --git a/src/main/java/gr/gousiosg/javacg/stat/DynamicCallManager.java b/src/main/java/gr/gousiosg/javacg/stat/DynamicCallManager.java new file mode 100644 index 00000000..5d1275cd --- /dev/null +++ b/src/main/java/gr/gousiosg/javacg/stat/DynamicCallManager.java @@ -0,0 +1,116 @@ +package gr.gousiosg.javacg.stat; + +import java.util.HashMap; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.bcel.classfile.Attribute; +import org.apache.bcel.classfile.BootstrapMethod; +import org.apache.bcel.classfile.BootstrapMethods; +import org.apache.bcel.classfile.ConstantMethodHandle; +import org.apache.bcel.classfile.ConstantMethodref; +import org.apache.bcel.classfile.ConstantNameAndType; +import org.apache.bcel.classfile.ConstantPool; +import org.apache.bcel.classfile.ConstantUtf8; +import org.apache.bcel.classfile.JavaClass; +import org.apache.bcel.classfile.Method; + +/** + * {@link DynamicCallManager} provides facilities to retrieve information about + * dynamic calls statically. + *

+ * Most of the time, call relationships are explicit, which allows to properly + * build the call graph statically. But in the case of dynamic linking, i.e. + * invokedynamic instructions, this relationship might be unknown + * until the code is actually executed. Indeed, bootstrap methods are used to + * dynamically link the code at first call. One can read details about the + * invokedynamic + * instruction to know more about this mechanism. + *

+ * Nested lambdas are particularly subject to such absence of concrete caller, + * which lead us to produce method names like lambda$null$0, which + * breaks the call graph. This information can however be retrieved statically + * through the code of the bootstrap method called. + *

+ * In {@link #retrieveCalls(Method, JavaClass)}, we retrieve the (called, + * caller) relationships by analyzing the code of the caller {@link Method}. + * This information is then used in {@link #linkCalls(Method)} to rename the + * called {@link Method} properly. + * + * @author Matthieu Vergne + * + */ +public class DynamicCallManager { + private static final Pattern BOOTSTRAP_CALL_PATTERN = Pattern + .compile("invokedynamic\t(\\d+):[^:]+:\\S+ \\(\\d+\\)"); + private static final int CALL_HANDLE_INDEX_ARGUMENT = 1; + + private final Map dynamicCallers = new HashMap<>(); + + /** + * Retrieve dynamic call relationships based on the code of the provided + * {@link Method}. + * + * @param method + * {@link Method} to analyze the code + * @param jc + * {@link JavaClass} info, which contains the bootstrap methods + * @see #linkCalls(Method) + */ + public void retrieveCalls(Method method, JavaClass jc) { + if (method.isAbstract()) { + // No code to consider + return; + } + ConstantPool cp = method.getConstantPool(); + BootstrapMethod[] boots = getBootstrapMethods(jc); + String code = method.getCode().toString(); + Matcher matcher = BOOTSTRAP_CALL_PATTERN.matcher(code); + while (matcher.find()) { + int bootIndex = Integer.parseInt(matcher.group(1)); + BootstrapMethod bootMethod = boots[bootIndex]; + int calledIndex = bootMethod.getBootstrapArguments()[CALL_HANDLE_INDEX_ARGUMENT]; + String calledName = getMethodNameFromHandleIndex(cp, calledIndex); + String callerName = method.getName(); + dynamicCallers.put(calledName, callerName); + } + } + + private String getMethodNameFromHandleIndex(ConstantPool cp, int callIndex) { + ConstantMethodHandle handle = (ConstantMethodHandle) cp.getConstant(callIndex); + ConstantMethodref ref = (ConstantMethodref) cp.getConstant(handle.getReferenceIndex()); + ConstantNameAndType nameAndType = (ConstantNameAndType) cp.getConstant(ref.getNameAndTypeIndex()); + return nameAndType.getName(cp); + } + + /** + * Link the {@link Method}'s name to its concrete caller if required. + * + * @param method + * {@link Method} to analyze + * @see #retrieveCalls(Method, JavaClass) + */ + public void linkCalls(Method method) { + int nameIndex = method.getNameIndex(); + ConstantPool cp = method.getConstantPool(); + String methodName = ((ConstantUtf8) cp.getConstant(nameIndex)).getBytes(); + String linkedName = methodName; + String callerName = methodName; + while (linkedName.matches("(lambda\\$)+null(\\$\\d+)+")) { + callerName = dynamicCallers.get(callerName); + linkedName = linkedName.replace("null", callerName); + } + cp.setConstant(nameIndex, new ConstantUtf8(linkedName)); + } + + private BootstrapMethod[] getBootstrapMethods(JavaClass jc) { + for (Attribute attribute : jc.getAttributes()) { + if (attribute instanceof BootstrapMethods) { + return ((BootstrapMethods) attribute).getBootstrapMethods(); + } + } + return new BootstrapMethod[] {}; + } +} From cd4c548d98601068cacae1afc0de9c51cb7d5ccc Mon Sep 17 00:00:00 2001 From: Matthieu Vergne Date: Mon, 22 Oct 2018 22:49:33 +0200 Subject: [PATCH 4/5] Fix: ConstantMethodref only is too narrow. Generalize from ConstantMethodref to ConstantCP to cover also ConstantInterfaceMethodref and possibly others. --- src/main/java/gr/gousiosg/javacg/stat/DynamicCallManager.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/gr/gousiosg/javacg/stat/DynamicCallManager.java b/src/main/java/gr/gousiosg/javacg/stat/DynamicCallManager.java index 5d1275cd..ef46a3ba 100644 --- a/src/main/java/gr/gousiosg/javacg/stat/DynamicCallManager.java +++ b/src/main/java/gr/gousiosg/javacg/stat/DynamicCallManager.java @@ -8,8 +8,8 @@ import org.apache.bcel.classfile.Attribute; import org.apache.bcel.classfile.BootstrapMethod; import org.apache.bcel.classfile.BootstrapMethods; +import org.apache.bcel.classfile.ConstantCP; import org.apache.bcel.classfile.ConstantMethodHandle; -import org.apache.bcel.classfile.ConstantMethodref; import org.apache.bcel.classfile.ConstantNameAndType; import org.apache.bcel.classfile.ConstantPool; import org.apache.bcel.classfile.ConstantUtf8; @@ -80,7 +80,7 @@ public void retrieveCalls(Method method, JavaClass jc) { private String getMethodNameFromHandleIndex(ConstantPool cp, int callIndex) { ConstantMethodHandle handle = (ConstantMethodHandle) cp.getConstant(callIndex); - ConstantMethodref ref = (ConstantMethodref) cp.getConstant(handle.getReferenceIndex()); + ConstantCP ref = (ConstantCP) cp.getConstant(handle.getReferenceIndex()); ConstantNameAndType nameAndType = (ConstantNameAndType) cp.getConstant(ref.getNameAndTypeIndex()); return nameAndType.getName(cp); } From e84db11ffbd3d75a6ba7e07e38648074f300dc93 Mon Sep 17 00:00:00 2001 From: Matthieu Vergne Date: Mon, 22 Oct 2018 22:51:03 +0200 Subject: [PATCH 5/5] Fix: BCEL bug fixed in 6.1. Bug: https://jira.apache.org/jira/browse/BCEL-286 Release note: https://www.apache.org/dist/commons/bcel/RELEASE-NOTES.txt --- pom.xml | 2 +- src/main/java/gr/gousiosg/javacg/stat/DynamicCallManager.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 3b510209..a606ecad 100644 --- a/pom.xml +++ b/pom.xml @@ -17,7 +17,7 @@ org.apache.bcel bcel - 6.0 + 6.2 provided diff --git a/src/main/java/gr/gousiosg/javacg/stat/DynamicCallManager.java b/src/main/java/gr/gousiosg/javacg/stat/DynamicCallManager.java index ef46a3ba..5a3bb0a3 100644 --- a/src/main/java/gr/gousiosg/javacg/stat/DynamicCallManager.java +++ b/src/main/java/gr/gousiosg/javacg/stat/DynamicCallManager.java @@ -44,7 +44,7 @@ */ public class DynamicCallManager { private static final Pattern BOOTSTRAP_CALL_PATTERN = Pattern - .compile("invokedynamic\t(\\d+):[^:]+:\\S+ \\(\\d+\\)"); + .compile("invokedynamic\t(\\d+):\\S+ \\S+ \\(\\d+\\)"); private static final int CALL_HANDLE_INDEX_ARGUMENT = 1; private final Map dynamicCallers = new HashMap<>();