diff --git a/README.md b/README.md index 1f2ac1bd..8f2b9c77 100644 --- a/README.md +++ b/README.md @@ -125,7 +125,23 @@ The output of all `runCoveragePerTestMethods()` API is a [`eu.stamp_project.test * `Map getCoverageResultsMap()`: returns a map that associate the simple of a test method to its instruction coverage. * `Coverage getCoverageOf(String testMethodName)`: returns the instruction coverage of a test method, specified by its simple name. - + +#### Covered results per test method + +In the same way, you can have the covered results per test method using `runCoveredTestResultPerTestMethods()` API of `EntryPoint` class. + +##### Output + +The output of all `runCoveredTestResultPerTestMethods()` API is a [`eu.stamp_project.testrunner.listener.CoveredTestResultPerTestMethod`](https://github.com/STAMP-project/testrunner/blob/master/src/main/java/eu/stamp_project/testrunner/listener/CoveragePerTestMethod.java#L13). + +* `Map getCoverageResultsMap()`: returns a map that associate the simple of a test method to its instruction coverage. +* `Coverage getCoverageOf(String testMethodName)`: returns the instruction coverage of a test method, specified by its simple name. +* `getRunningTests()`: returns the list of test methods that have been executed. +* `getPassingTests()`: returns the list of test methods that succeed. +* `getFailingTests()`: returns the list of test methods that failed. +* `getAssumptionFailingTests()`: returns the list of test methods that have a failing assumption. For example, in JUnit4 one can make assumptions using `org.junit.Assume` API, _e.g._ `Assume.assumeTrue(myBoolean)`. If the assumption does not hold, it is not necessary because the program is broken but rather than the test is irrelevant in the current state, _e.g._ one can make dedicated test to a platform. +* `getIgnoredTests()`: returns the list of test methods that are ignored. + #### Mutation Score The test runner can now compute the mutation using [PIT](http://pitest.org). diff --git a/src/main/java/eu/stamp_project/testrunner/EntryPoint.java b/src/main/java/eu/stamp_project/testrunner/EntryPoint.java index 5b4978ec..4581d6f1 100644 --- a/src/main/java/eu/stamp_project/testrunner/EntryPoint.java +++ b/src/main/java/eu/stamp_project/testrunner/EntryPoint.java @@ -4,6 +4,7 @@ import eu.stamp_project.testrunner.listener.*; import eu.stamp_project.testrunner.listener.impl.CoverageImpl; import eu.stamp_project.testrunner.listener.impl.CoveragePerTestMethodImpl; +import eu.stamp_project.testrunner.listener.impl.CoveredTestResultPerTestMethodImpl; import eu.stamp_project.testrunner.listener.impl.TestResultImpl; import eu.stamp_project.testrunner.listener.junit4.JUnit4Coverage; import eu.stamp_project.testrunner.listener.pit.AbstractParser; @@ -420,6 +421,91 @@ public static CoveragePerTestMethod runCoveragePerTestMethods(String classpath, return load; } + /* COMPUTE COVERED TEST RESULT PER TEST METHOD API */ + + public static CoveredTestResultPerTestMethod runCoveredTestResultPerTestMethods(String classpath, String targetProjectClasses, + String fullQualifiedNameOfTestClass) throws TimeoutException { + return EntryPoint.runCoveredTestResultPerTestMethods(classpath, targetProjectClasses, + new String[]{fullQualifiedNameOfTestClass}, new String[0]); + } + + public static CoveredTestResultPerTestMethod runCoveredTestResultPerTestMethods(String classpath, String targetProjectClasses, + String fullQualifiedNameOfTestClass, String testMethodName) throws TimeoutException { + return EntryPoint.runCoveredTestResultPerTestMethods(classpath, targetProjectClasses, + new String[]{fullQualifiedNameOfTestClass}, new String[]{testMethodName}); + } + + public static CoveredTestResultPerTestMethod runCoveredTestResultPerTestMethods(String classpath, String targetProjectClasses, + String fullQualifiedNameOfTestClass, String[] testMethodNames) throws TimeoutException { + return EntryPoint.runCoveredTestResultPerTestMethods(classpath, targetProjectClasses, + new String[]{fullQualifiedNameOfTestClass}, testMethodNames); + } + + public static CoveredTestResultPerTestMethod runCoveredTestResultPerTestMethods(String classpath, String targetProjectClasses, + String[] fullQualifiedNameOfTestClasses) throws TimeoutException { + return EntryPoint.runCoveredTestResultPerTestMethods(classpath, targetProjectClasses, fullQualifiedNameOfTestClasses, + new String[0]); + } + + /** + * Compute the test result and instruction coverage using JaCoCo for various test methods + * inside the given test classes. + *

+ * This method computes the instruction coverage, using JaCoCo obtained by executing the + * given test methods inside the given test classes. This method requires the + * path to the binaries, i.e. .class, of the source code on which the + * instruction must be computed. This method computes the per test method + * coverage, i.e. the coverage obtained by each test methods, separately. + * It does not run one by one test methods, but rather use a specific + * implementation of {@link org.junit.runner.notification.RunListener}. + *

+ * + * @param classpath the classpath required to run the given tests classes. + * @param targetProjectClasses path to the folder that contains binaries, i.e. .class, on which + * Jacoco computes the coverage. + * @param fullQualifiedNameOfTestClasses test classes to be run. + * @param methodNames test methods to be run. + * @return a Map that associate each test method name to its instruction + * coverage, as an instance of CoveredTestResultPerTestMethod {@link CoveredTestResultPerTestMethod} of + * test classes. + * @throws TimeoutException when the execution takes longer than timeoutInMs + */ + public static CoveredTestResultPerTestMethod runCoveredTestResultPerTestMethods(String classpath, String targetProjectClasses, + String[] fullQualifiedNameOfTestClasses, String[] methodNames) throws TimeoutException { + final String javaCommand = String.join(ConstantsHelper.WHITE_SPACE, + new String[]{ + getJavaCommand(), + (classpath + ConstantsHelper.PATH_SEPARATOR + ABSOLUTE_PATH_TO_RUNNER_CLASSES + + ConstantsHelper.PATH_SEPARATOR + ABSOLUTE_PATH_TO_JACOCO_DEPENDENCIES).replaceAll(" ", "%20"), + EntryPoint.jUnit5Mode ? EntryPoint.JUNIT5_JACOCO_RUNNER_COVERED_RESULT_PER_TEST_QUALIFIED_NAME : EntryPoint.JUNIT4_JACOCO_RUNNER_COVERED_RESULT_PER_TEST_QUALIFIED_NAME, + ParserOptions.FLAG_pathToCompiledClassesOfTheProject, + (targetProjectClasses).replaceAll(" ", "%20"), ParserOptions.FLAG_fullQualifiedNameOfTestClassToRun, + String.join(ConstantsHelper.PATH_SEPARATOR, fullQualifiedNameOfTestClasses), + methodNames.length == 0 ? "" : ParserOptions.FLAG_testMethodNamesToRun + ConstantsHelper.WHITE_SPACE + + String.join(ConstantsHelper.PATH_SEPARATOR, methodNames), + EntryPoint.blackList.isEmpty() ? "" + : (ParserOptions.FLAG_blackList + ConstantsHelper.WHITE_SPACE + + String.join(ConstantsHelper.PATH_SEPARATOR, EntryPoint.blackList)), + EntryPoint.coverageDetail == ParserOptions.CoverageTransformerDetail.SUMMARIZED ? "" : + (ParserOptions.FLAG_coverage_detail + ConstantsHelper.WHITE_SPACE + + EntryPoint.coverageDetail.name()), + }); + try { + EntryPoint.runGivenCommandLine(javaCommand); + } catch (TimeoutException e) { + LOGGER.warn("Timeout when running {}", javaCommand); + throw e; + } + final CoveredTestResultPerTestMethod load = CoveredTestResultPerTestMethodImpl.load(); + if (EntryPoint.verbose) { + LOGGER.info("Coverage per test methods has been computed {}{}", ConstantsHelper.LINE_SEPARATOR, + load.toString()); + } + return load; + } + /* COMPUTE MUTATION SCORE WITH PIT API */ /** @@ -575,6 +661,10 @@ private static void reset() { private static final String JUNIT5_JACOCO_RUNNER_PER_TEST_QUALIFIED_NAME = "eu.stamp_project.testrunner.runner.coverage.JUnit5JacocoRunnerPerTestMethod"; + private static final String JUNIT4_JACOCO_RUNNER_COVERED_RESULT_PER_TEST_QUALIFIED_NAME = "eu.stamp_project.testrunner.runner.coverage.JUnit4JacocoRunnerCoveredResultPerTestMethod"; + + private static final String JUNIT5_JACOCO_RUNNER_COVERED_RESULT_PER_TEST_QUALIFIED_NAME = "eu.stamp_project.testrunner.runner.coverage.JUnit5JacocoRunnerCoveredResultPerTestMethod"; + private static final String ABSOLUTE_PATH_TO_RUNNER_CLASSES = initAbsolutePathToRunnerClasses(); private static final int DEFAULT_TIMEOUT = 10000; diff --git a/src/main/java/eu/stamp_project/testrunner/listener/CoveredTestResultPerTestMethod.java b/src/main/java/eu/stamp_project/testrunner/listener/CoveredTestResultPerTestMethod.java new file mode 100644 index 00000000..bbb129b2 --- /dev/null +++ b/src/main/java/eu/stamp_project/testrunner/listener/CoveredTestResultPerTestMethod.java @@ -0,0 +1,28 @@ +package eu.stamp_project.testrunner.listener; + +import eu.stamp_project.testrunner.utils.ConstantsHelper; + +import java.io.Serializable; +import java.util.Map; + +/** + * Stores both the individual test unit (i.e. method) results as well as the individual coverage + * information for each of them. + * + * @author andre15silva + */ +public interface CoveredTestResultPerTestMethod extends TestResult, Serializable { + + public static final String SERIALIZE_NAME = "CoveredTestResultPerTest"; + + public static final String OUTPUT_DIR = "target" + ConstantsHelper.FILE_SEPARATOR; + + public static final String EXTENSION = ".ser"; + + public Map getCoverageResultsMap(); + + public Coverage getCoverageOf(String testMethodName); + + public void save(); + +} diff --git a/src/main/java/eu/stamp_project/testrunner/listener/impl/CoverageDetailed.java b/src/main/java/eu/stamp_project/testrunner/listener/impl/CoverageDetailed.java index a409e3fd..16b4826a 100644 --- a/src/main/java/eu/stamp_project/testrunner/listener/impl/CoverageDetailed.java +++ b/src/main/java/eu/stamp_project/testrunner/listener/impl/CoverageDetailed.java @@ -49,7 +49,7 @@ public boolean isBetterThan(Coverage that) { @Override public String toString() { - return null;// return this.instructionsCovered + " / " + this.instructionsTotal; + return covered.toString(); } @Override diff --git a/src/main/java/eu/stamp_project/testrunner/listener/impl/CoverageFromClass.java b/src/main/java/eu/stamp_project/testrunner/listener/impl/CoverageFromClass.java index 4769d394..76a2fc4e 100644 --- a/src/main/java/eu/stamp_project/testrunner/listener/impl/CoverageFromClass.java +++ b/src/main/java/eu/stamp_project/testrunner/listener/impl/CoverageFromClass.java @@ -1,5 +1,6 @@ package eu.stamp_project.testrunner.listener.impl; +import java.io.Serializable; import java.util.HashMap; import java.util.Map; @@ -8,7 +9,9 @@ * @author Matias Martinez * */ -public class CoverageFromClass { +public class CoverageFromClass implements Serializable { + + private static final long serialVersionUID = 4955729621536496728L; private String classname; private String packageName; diff --git a/src/main/java/eu/stamp_project/testrunner/listener/impl/CoveredTestResultPerTestMethodImpl.java b/src/main/java/eu/stamp_project/testrunner/listener/impl/CoveredTestResultPerTestMethodImpl.java new file mode 100644 index 00000000..d9dc72a7 --- /dev/null +++ b/src/main/java/eu/stamp_project/testrunner/listener/impl/CoveredTestResultPerTestMethodImpl.java @@ -0,0 +1,207 @@ +package eu.stamp_project.testrunner.listener.impl; + +import eu.stamp_project.testrunner.listener.Coverage; +import eu.stamp_project.testrunner.listener.CoverageTransformer; +import eu.stamp_project.testrunner.listener.CoveredTestResultPerTestMethod; +import eu.stamp_project.testrunner.listener.TestResult; +import eu.stamp_project.testrunner.runner.Failure; +import eu.stamp_project.testrunner.runner.Loader; +import eu.stamp_project.testrunner.utils.ConstantsHelper; +import org.jacoco.core.data.ExecutionDataStore; +import org.jacoco.core.data.SessionInfoStore; +import org.jacoco.core.runtime.RuntimeData; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.ObjectOutputStream; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * Default implementation of the CoveredTestResultPerTestMethod interface + * + * @author andre15silva + */ +public class CoveredTestResultPerTestMethodImpl implements CoveredTestResultPerTestMethod { + + private static final long serialVersionUID = -789740001022671146L; + + protected final Map coverageResultsMap; + + protected final String classesDirectory; + + protected transient RuntimeData data; + + protected transient ExecutionDataStore executionData; + + protected transient SessionInfoStore sessionInfos; + + protected transient CoverageTransformer coverageTransformer; + + private List runningTests; + private List failingTests; + private List assumptionFailingTests; + private List ignoredTests; + + public CoveredTestResultPerTestMethodImpl(RuntimeData data, String classesDirectory, CoverageTransformer coverageTransformer) { + this.data = data; + this.classesDirectory = classesDirectory; + this.coverageResultsMap = new HashMap<>(); + this.coverageTransformer = coverageTransformer; + this.runningTests = new ArrayList<>(); + this.failingTests = new ArrayList<>(); + this.assumptionFailingTests = new ArrayList<>(); + this.ignoredTests = new ArrayList<>(); + + } + + public String getClassesDirectory() { + return classesDirectory; + } + + public RuntimeData getData() { + return data; + } + + public ExecutionDataStore getExecutionData() { + return executionData; + } + + public SessionInfoStore getSessionInfos() { + return sessionInfos; + } + + public CoverageTransformer getCoverageTransformer() { + return coverageTransformer; + } + + public void setData(RuntimeData data) { + this.data = data; + } + + public void setExecutionData(ExecutionDataStore executionData) { + this.executionData = executionData; + } + + public void setSessionInfos(SessionInfoStore sessionInfos) { + this.sessionInfos = sessionInfos; + } + + @Override + public Map getCoverageResultsMap() { + return coverageResultsMap; + } + + @Override + public Coverage getCoverageOf(String testMethodName) { + return this.getCoverageResultsMap().get(testMethodName); + } + + @Override + public List getRunningTests() { + return runningTests; + } + + @Override + public List getPassingTests() { + final List failing = this.failingTests.stream() + .map(failure -> failure.testCaseName) + .collect(Collectors.toList()); + final List assumptionFailing = this.assumptionFailingTests.stream() + .map(failure -> failure.testCaseName) + .collect(Collectors.toList()); + return this.runningTests.stream() + .filter(description -> !assumptionFailing.contains(description)) + .filter(description -> !failing.contains(description)) + .collect(Collectors.toList()); + } + + @Override + public TestResult aggregate(TestResult that) { + if (that instanceof CoveredTestResultPerTestMethodImpl) { + final CoveredTestResultPerTestMethodImpl thatListener = (CoveredTestResultPerTestMethodImpl) that; + this.runningTests.addAll(thatListener.runningTests); + this.failingTests.addAll(thatListener.failingTests); + this.assumptionFailingTests.addAll(thatListener.assumptionFailingTests); + this.ignoredTests.addAll(thatListener.ignoredTests); + } + return this; + } + + @Override + public List getFailingTests() { + return failingTests; + } + + @Override + public List getAssumptionFailingTests() { + return assumptionFailingTests; + } + + @Override + public List getIgnoredTests() { + return ignoredTests; + } + + @Override + public Failure getFailureOf(String testMethodName) { + return this.getFailingTests().stream() + .filter(failure -> failure.testCaseName.equals(testMethodName)) + .findFirst() + .orElseThrow(() -> new IllegalArgumentException(String.format("Could not find %s in failing test", testMethodName))); + } + + @Override + public void save() { + File outputDir = new File(TestResult.OUTPUT_DIR); + if (!outputDir.exists()) { + if (!outputDir.mkdirs()) { + System.err.println("Error while creating output dir"); + } + } + File f = new File(outputDir, SERIALIZE_NAME + EXTENSION); + try (FileOutputStream fout = new FileOutputStream(f)) { + try (ObjectOutputStream oos = new ObjectOutputStream(fout)) { + oos.writeObject(this); + } catch (Exception e) { + throw new RuntimeException(e); + } + } catch (Exception e) { + System.err.println("Error while writing serialized file."); + throw new RuntimeException(e); + } + System.out.println("File saved to the following path: " + f.getAbsolutePath()); + } + + /** + * Load from serialized object + * + * @return an Instance of CoveragePerTestMethod loaded from a serialized file. + */ + public static CoveredTestResultPerTestMethodImpl load() { + return new Loader().load(SERIALIZE_NAME); + } + + @Override + public String toString() { + return "CoveredTestResultPerTestMethodImpl{" + + "coverageResultsMap=" + this.coverageResultsMap.keySet() + .stream() + .map(test -> "\t" + test + ": " + coverageResultsMap.get(test).toString()) + .collect(Collectors.joining(ConstantsHelper.LINE_SEPARATOR)) + + ", classesDirectory='" + classesDirectory + '\'' + + ", data=" + data + + ", executionData=" + executionData + + ", sessionInfos=" + sessionInfos + + ", coverageTransformer=" + coverageTransformer + + ", runningTests=" + runningTests + + ", failingTests=" + failingTests + + ", assumptionFailingTests=" + assumptionFailingTests + + ", ignoredTests=" + ignoredTests + + '}'; + } + +} diff --git a/src/main/java/eu/stamp_project/testrunner/listener/junit4/CoveredTestResultsPerJUnit4TestMethod.java b/src/main/java/eu/stamp_project/testrunner/listener/junit4/CoveredTestResultsPerJUnit4TestMethod.java new file mode 100644 index 00000000..5358ca17 --- /dev/null +++ b/src/main/java/eu/stamp_project/testrunner/listener/junit4/CoveredTestResultsPerJUnit4TestMethod.java @@ -0,0 +1,257 @@ +package eu.stamp_project.testrunner.listener.junit4; + +import eu.stamp_project.testrunner.listener.Coverage; +import eu.stamp_project.testrunner.listener.CoverageTransformer; +import eu.stamp_project.testrunner.listener.CoveredTestResultPerTestMethod; +import eu.stamp_project.testrunner.listener.TestResult; +import eu.stamp_project.testrunner.listener.impl.CoverageImpl; +import eu.stamp_project.testrunner.listener.impl.CoveredTestResultPerTestMethodImpl; +import eu.stamp_project.testrunner.runner.Failure; +import org.jacoco.core.analysis.Analyzer; +import org.jacoco.core.analysis.CoverageBuilder; +import org.jacoco.core.analysis.IClassCoverage; +import org.jacoco.core.analysis.ICounter; +import org.jacoco.core.data.ExecutionDataStore; +import org.jacoco.core.data.SessionInfoStore; +import org.jacoco.core.runtime.RuntimeData; +import org.junit.runner.Description; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.function.Predicate; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +/** + * JUnit4 implementation of {@link CoveredTestResultPerTestMethod} + * + * @author andre15silva + */ +public class CoveredTestResultsPerJUnit4TestMethod extends JUnit4TestResult implements CoveredTestResultPerTestMethod { + + private static final long serialVersionUID = -6934847896187177463L; + + private CoveredTestResultPerTestMethodImpl internalCoveredTestResult; + + /** + * This field is used to support parametrized test + * In fact, parametrized are reported as follow: + * - the test method is named "test" + * - it reports, for each parameter as follow: test[0], test[1]. + * This field stores every coverage, i.e. for each input. + * Then, we are able to aggregate them to obtain the coverage of the test, for EVERY input. + */ + private Map> coveragesPerMethodName; + + public CoveredTestResultsPerJUnit4TestMethod(RuntimeData data, String classesDirectory, CoverageTransformer coverageTransformer) { + this.internalCoveredTestResult = new CoveredTestResultPerTestMethodImpl(data, classesDirectory, coverageTransformer); + this.coveragesPerMethodName = new HashMap<>(); + } + + private static final Predicate isParametrized = testMethodName -> + Pattern.compile(".+\\[\\d+\\]").matcher(testMethodName).matches(); + + private static final Function fromParametrizedToSimpleName = parametrizedName -> + parametrizedName.contains("[") ? parametrizedName.split("\\[")[0] : parametrizedName; + + @Override + public synchronized void testStarted(Description description) throws Exception { + this.internalCoveredTestResult.setExecutionData(new ExecutionDataStore()); + this.internalCoveredTestResult.setSessionInfos(new SessionInfoStore()); + this.internalCoveredTestResult.getData().setSessionId(description.getMethodName()); + this.internalCoveredTestResult.getData().collect( + this.internalCoveredTestResult.getExecutionData(), + this.internalCoveredTestResult.getSessionInfos(), + true + ); + } + + @Override + public synchronized void testFinished(Description description) throws Exception { + this.internalCoveredTestResult.getRunningTests().add(description.getMethodName()); + + this.internalCoveredTestResult.getData().collect( + this.internalCoveredTestResult.getExecutionData(), + this.internalCoveredTestResult.getSessionInfos(), + false + ); + + Coverage jUnit4Coverage = + internalCoveredTestResult.getCoverageTransformer().transformJacocoObject(this.internalCoveredTestResult.getExecutionData(), + this.internalCoveredTestResult.getClassesDirectory()); + this.internalCoveredTestResult.getCoverageResultsMap().put(description.getMethodName(), jUnit4Coverage); + if (isParametrized.test(description.getMethodName())) { + this.collectForParametrizedTest(fromParametrizedToSimpleName.apply(description.getMethodName())); + } + } + + @Override + public void testFailure(org.junit.runner.notification.Failure failure) throws Exception { + this.internalCoveredTestResult.getFailingTests().add( + new Failure( + failure.getDescription().getMethodName(), + failure.getDescription().getClassName(), + failure.getException() + ) + ); + } + + @Override + public void testAssumptionFailure(org.junit.runner.notification.Failure failure) { + this.internalCoveredTestResult.getAssumptionFailingTests().add( + new Failure( + failure.getDescription().getMethodName(), + failure.getDescription().getClassName(), + failure.getException() + ) + ); + } + + @Override + public void testIgnored(Description description) throws Exception { + this.internalCoveredTestResult.getIgnoredTests().add(description.getMethodName()); + } + + private void collectForParametrizedTest(String testMethodName) { + final CoverageBuilder coverageBuilder = new CoverageBuilder(); + final Analyzer analyzer = new Analyzer(this.internalCoveredTestResult.getExecutionData(), coverageBuilder); + try { + analyzer.analyzeAll(new File(this.internalCoveredTestResult.getClassesDirectory())); + } catch (IOException e) { + throw new RuntimeException(e); + } + if (!this.coveragesPerMethodName.containsKey(testMethodName)) { + this.coveragesPerMethodName.put(testMethodName, new ArrayList<>()); + } + coverageBuilder.getClasses() + .forEach(classCoverage -> + this.coveragesPerMethodName.get(testMethodName) + .add(classCoverage) + ); + } + + public Map> getCoveragesPerMethodName() { + return coveragesPerMethodName; + } + + @Override + public Map getCoverageResultsMap() { + return this.internalCoveredTestResult.getCoverageResultsMap(); + } + + @Override + public Coverage getCoverageOf(String testMethodName) { + return this.internalCoveredTestResult.getCoverageOf(testMethodName); + } + + @Override + public List getRunningTests() { + return this.internalCoveredTestResult.getRunningTests(); + } + + @Override + public List getPassingTests() { + return this.internalCoveredTestResult.getPassingTests(); + } + + @Override + public TestResult aggregate(TestResult that) { + if (that instanceof CoveredTestResultsPerJUnit4TestMethod) { + return this.internalCoveredTestResult.aggregate(((CoveredTestResultsPerJUnit4TestMethod) that).internalCoveredTestResult); + } else { + return this; + } + } + + @Override + public List getFailingTests() { + return this.internalCoveredTestResult.getFailingTests(); + } + + @Override + public List getAssumptionFailingTests() { + return this.internalCoveredTestResult.getAssumptionFailingTests(); + } + + @Override + public List getIgnoredTests() { + return this.internalCoveredTestResult.getIgnoredTests(); + } + + @Override + public Failure getFailureOf(String testMethodName) { + return this.getFailingTests().stream() + .filter(failure -> failure.testCaseName.equals(testMethodName)) + .findFirst() + .orElseThrow(() -> new IllegalArgumentException(String.format("Could not find %s in failing test", testMethodName))); + } + + @Override + public void save() { + if (!this.coveragesPerMethodName.isEmpty()) { + this.aggregateParametrizedTestCoverage(); + } + this.internalCoveredTestResult.save(); + } + + /* + This method will use the coveragesPerMethodName map to create a coverage for the test method. + This coverage is the aggregation of the coverages of each input for the same test method. + */ + private void aggregateParametrizedTestCoverage() { + this.coveragesPerMethodName.keySet().forEach(testMethodName -> { + int covered = 0; + int total = 0; + final ArrayList classCoverages = + new ArrayList<>(this.coveragesPerMethodName.get(testMethodName)); + while (!classCoverages.isEmpty()) { + final IClassCoverage current = classCoverages.get(0); + final List subListOnSameClass = + getSameClassCoverage(current.getName(), classCoverages); + final List> coveragePerMethods = subListOnSameClass.stream() + .map(coverage -> + CoverageImpl.getListOfCountForCounterFunction(coverage, ICounter::getCoveredCount) + ).collect(Collectors.toList()); + final List bestCoverage = IntStream.range(0, coveragePerMethods.get(0).size()) + .boxed() + .map(index -> + coveragePerMethods.stream() + .map(integers -> integers.get(index)) + .max(Integer::compareTo) + .get() + ).collect(Collectors.toList()); + covered += bestCoverage.stream().mapToInt(Integer::intValue).sum(); + total += CoverageImpl.getListOfCountForCounterFunction(subListOnSameClass.get(0), ICounter::getTotalCount) + .stream() + .mapToInt(Integer::intValue) + .sum(); + classCoverages.removeAll(subListOnSameClass); + } + final JUnit4Coverage value = new JUnit4Coverage(covered, total); + value.setExecutionPath(this.internalCoveredTestResult.getCoverageResultsMap().keySet() + .stream() + .map(this.internalCoveredTestResult.getCoverageResultsMap()::get) + .map(Coverage::getExecutionPath) + .collect(Collectors.joining("#")) + ); + this.internalCoveredTestResult.getCoverageResultsMap().put(testMethodName, value); + } + ); + } + + /* + * return the sublist of the given list which the coverage of the same class, pointed by the className + */ + private List getSameClassCoverage(String className, List classCoverages) { + return classCoverages.stream() + .filter(iClassCoverage -> className.equals(iClassCoverage.getName())) + .collect(Collectors.toList()); + } + +} \ No newline at end of file diff --git a/src/main/java/eu/stamp_project/testrunner/listener/junit5/CoveredTestResultsPerJUnit5TestMethod.java b/src/main/java/eu/stamp_project/testrunner/listener/junit5/CoveredTestResultsPerJUnit5TestMethod.java new file mode 100644 index 00000000..1f05f213 --- /dev/null +++ b/src/main/java/eu/stamp_project/testrunner/listener/junit5/CoveredTestResultsPerJUnit5TestMethod.java @@ -0,0 +1,96 @@ +package eu.stamp_project.testrunner.listener.junit5; + +import eu.stamp_project.testrunner.listener.Coverage; +import eu.stamp_project.testrunner.listener.CoverageTransformer; +import eu.stamp_project.testrunner.listener.CoveredTestResultPerTestMethod; +import eu.stamp_project.testrunner.listener.impl.CoveredTestResultPerTestMethodImpl; +import eu.stamp_project.testrunner.runner.Failure; +import org.jacoco.core.data.ExecutionDataStore; +import org.jacoco.core.data.SessionInfoStore; +import org.jacoco.core.runtime.RuntimeData; +import org.junit.platform.engine.TestExecutionResult; +import org.junit.platform.launcher.TestIdentifier; + +import java.util.Map; + +/** + * JUnit5 implementation of {@link CoveredTestResultPerTestMethod} + * + * @author andre15silva + */ +public class CoveredTestResultsPerJUnit5TestMethod extends JUnit5TestResult implements CoveredTestResultPerTestMethod { + + private static final long serialVersionUID = 5003707306546430948L; + + private CoveredTestResultPerTestMethodImpl internalCoveredTestResult; + + public CoveredTestResultsPerJUnit5TestMethod(RuntimeData data, String classesDirectory, CoverageTransformer coverageTransformer) { + this.internalCoveredTestResult = new CoveredTestResultPerTestMethodImpl(data, classesDirectory, coverageTransformer); + } + + @Override + public void executionSkipped(TestIdentifier testIdentifier, String reason) { + if (testIdentifier.isTest()) { + this.internalCoveredTestResult.getIgnoredTests().add(this.toString.apply(testIdentifier)); + } + } + + @Override + public synchronized void executionStarted(TestIdentifier testIdentifier) { + if (testIdentifier.isTest()) { + this.internalCoveredTestResult.setExecutionData(new ExecutionDataStore()); + this.internalCoveredTestResult.setSessionInfos(new SessionInfoStore()); + this.internalCoveredTestResult.getData().setSessionId(this.toString.apply(testIdentifier)); + this.internalCoveredTestResult.getData().collect( + this.internalCoveredTestResult.getExecutionData(), + this.internalCoveredTestResult.getSessionInfos(), + true + ); + this.internalCoveredTestResult.getRunningTests().add(this.toString.apply(testIdentifier)); + } + } + + @Override + public synchronized void executionFinished(TestIdentifier testIdentifier, TestExecutionResult testExecutionResult) { + if (testIdentifier.isTest()) { + this.internalCoveredTestResult.getData().collect( + this.internalCoveredTestResult.getExecutionData(), + this.internalCoveredTestResult.getSessionInfos(), + false + ); + + Coverage jUnit5Coverage = + internalCoveredTestResult.getCoverageTransformer().transformJacocoObject(this.internalCoveredTestResult.getExecutionData(), + this.internalCoveredTestResult.getClassesDirectory()); + this.internalCoveredTestResult.getCoverageResultsMap().put(this.toString.apply(testIdentifier), jUnit5Coverage); + switch (testExecutionResult.getStatus()) { + case FAILED: + this.internalCoveredTestResult.getFailingTests().add( + new Failure( + this.toString.apply(testIdentifier), + this.toClassName.apply(testIdentifier), + testExecutionResult.getThrowable().get() + ) + ); + break; + default: + break; + } + } + } + + @Override + public Map getCoverageResultsMap() { + return this.internalCoveredTestResult.getCoverageResultsMap(); + } + + @Override + public Coverage getCoverageOf(String testMethodName) { + return this.internalCoveredTestResult.getCoverageOf(testMethodName); + } + + @Override + public void save() { + this.internalCoveredTestResult.save(); + } +} diff --git a/src/main/java/eu/stamp_project/testrunner/runner/coverage/JUnit4JacocoRunnerCoveredResultPerTestMethod.java b/src/main/java/eu/stamp_project/testrunner/runner/coverage/JUnit4JacocoRunnerCoveredResultPerTestMethod.java new file mode 100644 index 00000000..38e1f4d5 --- /dev/null +++ b/src/main/java/eu/stamp_project/testrunner/runner/coverage/JUnit4JacocoRunnerCoveredResultPerTestMethod.java @@ -0,0 +1,60 @@ +package eu.stamp_project.testrunner.runner.coverage; + +import eu.stamp_project.testrunner.listener.CoverageTransformer; +import eu.stamp_project.testrunner.listener.CoveredTestResultPerTestMethod; +import eu.stamp_project.testrunner.listener.junit4.CoveredTestResultsPerJUnit4TestMethod; +import eu.stamp_project.testrunner.listener.junit4.JUnit4TestResult; +import eu.stamp_project.testrunner.runner.JUnit4Runner; +import eu.stamp_project.testrunner.runner.ParserOptions; +import eu.stamp_project.testrunner.utils.ConstantsHelper; +import org.jacoco.core.runtime.RuntimeData; + +import java.util.Collections; +import java.util.List; + +/** + * JUnit4 implementation of {@link JacocoRunnerCoveredResultPerTestMethod}. + * + * @author andre15silva + */ +public class JUnit4JacocoRunnerCoveredResultPerTestMethod extends JacocoRunnerCoveredResultPerTestMethod { + + public JUnit4JacocoRunnerCoveredResultPerTestMethod(String classesDirectory, String testClassesDirectory, CoverageTransformer coverageTransformer) { + super(classesDirectory, testClassesDirectory, coverageTransformer); + } + + public JUnit4JacocoRunnerCoveredResultPerTestMethod(String classesDirectory, String testClassesDirectory, List blackList, CoverageTransformer coverageTransformer) { + super(classesDirectory, testClassesDirectory, blackList, coverageTransformer); + } + + @Override + protected CoveredTestResultPerTestMethod executeCoveredTestPerTestMethod(RuntimeData data, String classesDirectory, String[] testClassNames, String[] testMethodNames) { + final CoveredTestResultsPerJUnit4TestMethod listener = new CoveredTestResultsPerJUnit4TestMethod(data, classesDirectory, coverageTransformer); + JUnit4Runner.run( + testClassNames, + testMethodNames, + Collections.emptyList(), + (JUnit4TestResult) listener, + this.instrumentedClassLoader + ); + return listener; + } + + public static void main(String[] args) { + final ParserOptions options = ParserOptions.parse(args); + final String[] splittedArgs0 = options.getPathToCompiledClassesOfTheProject().split(ConstantsHelper.PATH_SEPARATOR); + final String classesDirectory = splittedArgs0[0]; + final String testClassesDirectory = splittedArgs0[1]; + new JUnit4JacocoRunnerCoveredResultPerTestMethod( + classesDirectory, + testClassesDirectory, + options.getBlackList(), + options.getCoverageTransformer() + ).runCoveredTestResultPerTestMethod(classesDirectory, + testClassesDirectory, + options.getFullQualifiedNameOfTestClassesToRun()[0], + options.getTestMethodNamesToRun() + ).save(); + } + +} diff --git a/src/main/java/eu/stamp_project/testrunner/runner/coverage/JUnit5JacocoRunnerCoveredResultPerTestMethod.java b/src/main/java/eu/stamp_project/testrunner/runner/coverage/JUnit5JacocoRunnerCoveredResultPerTestMethod.java new file mode 100644 index 00000000..ad4e868c --- /dev/null +++ b/src/main/java/eu/stamp_project/testrunner/runner/coverage/JUnit5JacocoRunnerCoveredResultPerTestMethod.java @@ -0,0 +1,60 @@ +package eu.stamp_project.testrunner.runner.coverage; + +import eu.stamp_project.testrunner.listener.CoverageTransformer; +import eu.stamp_project.testrunner.listener.CoveredTestResultPerTestMethod; +import eu.stamp_project.testrunner.listener.junit5.CoveredTestResultsPerJUnit5TestMethod; +import eu.stamp_project.testrunner.listener.junit5.JUnit5TestResult; +import eu.stamp_project.testrunner.runner.JUnit5Runner; +import eu.stamp_project.testrunner.runner.ParserOptions; +import eu.stamp_project.testrunner.utils.ConstantsHelper; +import org.jacoco.core.runtime.RuntimeData; + +import java.util.Collections; +import java.util.List; + +/** + * JUnit5 implementation of {@link JacocoRunnerCoveredResultPerTestMethod}. + * + * @author andre15silva + */ +public class JUnit5JacocoRunnerCoveredResultPerTestMethod extends JacocoRunnerCoveredResultPerTestMethod { + + public JUnit5JacocoRunnerCoveredResultPerTestMethod(String classesDirectory, String testClassesDirectory, CoverageTransformer coverageTransformer) { + super(classesDirectory, testClassesDirectory, coverageTransformer); + } + + public JUnit5JacocoRunnerCoveredResultPerTestMethod(String classesDirectory, String testClassesDirectory, List blackList, CoverageTransformer coverageTransformer) { + super(classesDirectory, testClassesDirectory, blackList, coverageTransformer); + } + + @Override + protected CoveredTestResultPerTestMethod executeCoveredTestPerTestMethod(RuntimeData data, String classesDirectory, String[] testClassNames, String[] testMethodNames) { + final CoveredTestResultsPerJUnit5TestMethod listener = new CoveredTestResultsPerJUnit5TestMethod(data, classesDirectory, coverageTransformer); + JUnit5Runner.run( + testClassNames, + testMethodNames, + Collections.emptyList(), + (JUnit5TestResult) listener, + this.instrumentedClassLoader + ); + return listener; + } + + public static void main(String[] args) { + final ParserOptions options = ParserOptions.parse(args); + final String[] splittedArgs0 = options.getPathToCompiledClassesOfTheProject().split(ConstantsHelper.PATH_SEPARATOR); + final String classesDirectory = splittedArgs0[0]; + final String testClassesDirectory = splittedArgs0[1]; + new JUnit5JacocoRunnerCoveredResultPerTestMethod( + classesDirectory, + testClassesDirectory, + options.getBlackList(), + options.getCoverageTransformer() + ).runCoveredTestResultPerTestMethod(classesDirectory, + testClassesDirectory, + options.getFullQualifiedNameOfTestClassesToRun()[0], + options.getTestMethodNamesToRun() + ).save(); + } + +} diff --git a/src/main/java/eu/stamp_project/testrunner/runner/coverage/JacocoRunnerCoveredResultPerTestMethod.java b/src/main/java/eu/stamp_project/testrunner/runner/coverage/JacocoRunnerCoveredResultPerTestMethod.java new file mode 100644 index 00000000..423c171c --- /dev/null +++ b/src/main/java/eu/stamp_project/testrunner/runner/coverage/JacocoRunnerCoveredResultPerTestMethod.java @@ -0,0 +1,83 @@ +package eu.stamp_project.testrunner.runner.coverage; + +import eu.stamp_project.testrunner.listener.CoveragePerTestMethod; +import eu.stamp_project.testrunner.listener.CoverageTransformer; +import eu.stamp_project.testrunner.listener.CoveredTestResultPerTestMethod; +import eu.stamp_project.testrunner.listener.TestResult; +import eu.stamp_project.testrunner.runner.Failure; +import eu.stamp_project.testrunner.utils.ConstantsHelper; +import org.apache.commons.io.IOUtils; +import org.jacoco.core.runtime.RuntimeData; + +import java.io.File; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.URLClassLoader; +import java.util.List; +import java.util.stream.Collectors; + +import static java.util.ResourceBundle.clearCache; + +/** + * Runs {@link JacocoRunnerPerTestMethod} but with a {@link CoveredTestResultPerTestMethod} listener + * + * @author andre15silva + */ +public abstract class JacocoRunnerCoveredResultPerTestMethod extends JacocoRunnerPerTestMethod { + + + public JacocoRunnerCoveredResultPerTestMethod(String classesDirectory, String testClassesDirectory, CoverageTransformer coverageTransformer) { + super(classesDirectory, testClassesDirectory, coverageTransformer); + } + + public JacocoRunnerCoveredResultPerTestMethod(String classesDirectory, String testClassesDirectory, List blackList, CoverageTransformer coverageTransformer) { + super(classesDirectory, testClassesDirectory, blackList, coverageTransformer); + } + + public CoveredTestResultPerTestMethod runCoveredTestResultPerTestMethod(String classesDirectory, + String testClassesDirectory, + String fullQualifiedNameOfTestClass, + String[] testMethodNames) { + final RuntimeData data = new RuntimeData(); + URLClassLoader classLoader; + try { + classLoader = new URLClassLoader(new URL[] + {new File(testClassesDirectory).toURI().toURL()}, this.instrumentedClassLoader); + } catch (MalformedURLException e) { + throw new RuntimeException(e); + } + final String resource = ConstantsHelper.fullQualifiedNameToPath.apply(fullQualifiedNameOfTestClass) + ".class"; + try { + this.instrumentedClassLoader.addDefinition( + fullQualifiedNameOfTestClass, + IOUtils.toByteArray(classLoader.getResourceAsStream(resource)) + ); + this.runtime.startup(data); + final CoveredTestResultPerTestMethod listener = this.executeCoveredTestPerTestMethod(data, classesDirectory, new String[]{fullQualifiedNameOfTestClass}, testMethodNames); + if (!((TestResult) listener).getFailingTests().isEmpty()) { + System.err.println("Some test(s) failed during computation of coverage:\n" + + ((TestResult) listener).getFailingTests() + .stream() + .map(Failure::toString) + .collect(Collectors.joining("\n")) + ); + } + this.runtime.shutdown(); + clearCache(this.instrumentedClassLoader); + return listener; + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Override + protected CoveragePerTestMethod executeTestPerTestMethod(RuntimeData data, String classesDirectory, String[] testClassNames, String[] testMethodNames) { + throw new UnsupportedOperationException(); + } + + protected abstract CoveredTestResultPerTestMethod executeCoveredTestPerTestMethod(RuntimeData data, + String classesDirectory, + String[] testClassNames, + String[] testMethodNames); + +} \ No newline at end of file diff --git a/src/main/resources/runner-classes/eu/stamp_project/testrunner/listener/Coverage.class b/src/main/resources/runner-classes/eu/stamp_project/testrunner/listener/Coverage.class index 07d7df13..e1fa36be 100644 Binary files a/src/main/resources/runner-classes/eu/stamp_project/testrunner/listener/Coverage.class and b/src/main/resources/runner-classes/eu/stamp_project/testrunner/listener/Coverage.class differ diff --git a/src/main/resources/runner-classes/eu/stamp_project/testrunner/listener/TestResult.class b/src/main/resources/runner-classes/eu/stamp_project/testrunner/listener/TestResult.class index 64aab0fe..3f550a9e 100644 Binary files a/src/main/resources/runner-classes/eu/stamp_project/testrunner/listener/TestResult.class and b/src/main/resources/runner-classes/eu/stamp_project/testrunner/listener/TestResult.class differ diff --git a/src/main/resources/runner-classes/eu/stamp_project/testrunner/listener/impl/CoverageImpl.class b/src/main/resources/runner-classes/eu/stamp_project/testrunner/listener/impl/CoverageImpl.class index dcfe7b3a..02a4c0fb 100644 Binary files a/src/main/resources/runner-classes/eu/stamp_project/testrunner/listener/impl/CoverageImpl.class and b/src/main/resources/runner-classes/eu/stamp_project/testrunner/listener/impl/CoverageImpl.class differ diff --git a/src/main/resources/runner-classes/eu/stamp_project/testrunner/listener/impl/CoveragePerTestMethodImpl.class b/src/main/resources/runner-classes/eu/stamp_project/testrunner/listener/impl/CoveragePerTestMethodImpl.class index ba2b6e5a..9a8b964b 100644 Binary files a/src/main/resources/runner-classes/eu/stamp_project/testrunner/listener/impl/CoveragePerTestMethodImpl.class and b/src/main/resources/runner-classes/eu/stamp_project/testrunner/listener/impl/CoveragePerTestMethodImpl.class differ diff --git a/src/main/resources/runner-classes/eu/stamp_project/testrunner/listener/junit4/CoveragePerJUnit4TestMethod.class b/src/main/resources/runner-classes/eu/stamp_project/testrunner/listener/junit4/CoveragePerJUnit4TestMethod.class index 0f5c3f9a..22fead9f 100644 Binary files a/src/main/resources/runner-classes/eu/stamp_project/testrunner/listener/junit4/CoveragePerJUnit4TestMethod.class and b/src/main/resources/runner-classes/eu/stamp_project/testrunner/listener/junit4/CoveragePerJUnit4TestMethod.class differ diff --git a/src/main/resources/runner-classes/eu/stamp_project/testrunner/listener/junit4/JUnit4Coverage.class b/src/main/resources/runner-classes/eu/stamp_project/testrunner/listener/junit4/JUnit4Coverage.class index d4f00b71..164ee0c2 100644 Binary files a/src/main/resources/runner-classes/eu/stamp_project/testrunner/listener/junit4/JUnit4Coverage.class and b/src/main/resources/runner-classes/eu/stamp_project/testrunner/listener/junit4/JUnit4Coverage.class differ diff --git a/src/main/resources/runner-classes/eu/stamp_project/testrunner/listener/junit5/CoveragePerJUnit5TestMethod$1.class b/src/main/resources/runner-classes/eu/stamp_project/testrunner/listener/junit5/CoveragePerJUnit5TestMethod$1.class index 97bcc7da..c38afb6e 100644 Binary files a/src/main/resources/runner-classes/eu/stamp_project/testrunner/listener/junit5/CoveragePerJUnit5TestMethod$1.class and b/src/main/resources/runner-classes/eu/stamp_project/testrunner/listener/junit5/CoveragePerJUnit5TestMethod$1.class differ diff --git a/src/main/resources/runner-classes/eu/stamp_project/testrunner/listener/junit5/CoveragePerJUnit5TestMethod.class b/src/main/resources/runner-classes/eu/stamp_project/testrunner/listener/junit5/CoveragePerJUnit5TestMethod.class index b44f1f8a..3c937bab 100644 Binary files a/src/main/resources/runner-classes/eu/stamp_project/testrunner/listener/junit5/CoveragePerJUnit5TestMethod.class and b/src/main/resources/runner-classes/eu/stamp_project/testrunner/listener/junit5/CoveragePerJUnit5TestMethod.class differ diff --git a/src/main/resources/runner-classes/eu/stamp_project/testrunner/listener/junit5/JUnit5Coverage.class b/src/main/resources/runner-classes/eu/stamp_project/testrunner/listener/junit5/JUnit5Coverage.class index 27782ff3..8f47851c 100644 Binary files a/src/main/resources/runner-classes/eu/stamp_project/testrunner/listener/junit5/JUnit5Coverage.class and b/src/main/resources/runner-classes/eu/stamp_project/testrunner/listener/junit5/JUnit5Coverage.class differ diff --git a/src/main/resources/runner-classes/eu/stamp_project/testrunner/runner/ParserOptions.class b/src/main/resources/runner-classes/eu/stamp_project/testrunner/runner/ParserOptions.class index 68476591..bd86f238 100644 Binary files a/src/main/resources/runner-classes/eu/stamp_project/testrunner/runner/ParserOptions.class and b/src/main/resources/runner-classes/eu/stamp_project/testrunner/runner/ParserOptions.class differ diff --git a/src/main/resources/runner-classes/eu/stamp_project/testrunner/runner/coverage/JUnit4JacocoRunner.class b/src/main/resources/runner-classes/eu/stamp_project/testrunner/runner/coverage/JUnit4JacocoRunner.class index fcfab49d..63158641 100644 Binary files a/src/main/resources/runner-classes/eu/stamp_project/testrunner/runner/coverage/JUnit4JacocoRunner.class and b/src/main/resources/runner-classes/eu/stamp_project/testrunner/runner/coverage/JUnit4JacocoRunner.class differ diff --git a/src/main/resources/runner-classes/eu/stamp_project/testrunner/runner/coverage/JUnit4JacocoRunnerPerTestMethod.class b/src/main/resources/runner-classes/eu/stamp_project/testrunner/runner/coverage/JUnit4JacocoRunnerPerTestMethod.class index 5c04e8c1..87207c2d 100644 Binary files a/src/main/resources/runner-classes/eu/stamp_project/testrunner/runner/coverage/JUnit4JacocoRunnerPerTestMethod.class and b/src/main/resources/runner-classes/eu/stamp_project/testrunner/runner/coverage/JUnit4JacocoRunnerPerTestMethod.class differ diff --git a/src/main/resources/runner-classes/eu/stamp_project/testrunner/runner/coverage/JUnit5JacocoRunner.class b/src/main/resources/runner-classes/eu/stamp_project/testrunner/runner/coverage/JUnit5JacocoRunner.class index e2c3a723..5b31c7fd 100644 Binary files a/src/main/resources/runner-classes/eu/stamp_project/testrunner/runner/coverage/JUnit5JacocoRunner.class and b/src/main/resources/runner-classes/eu/stamp_project/testrunner/runner/coverage/JUnit5JacocoRunner.class differ diff --git a/src/main/resources/runner-classes/eu/stamp_project/testrunner/runner/coverage/JUnit5JacocoRunnerPerTestMethod.class b/src/main/resources/runner-classes/eu/stamp_project/testrunner/runner/coverage/JUnit5JacocoRunnerPerTestMethod.class index cbc58da7..a320c003 100644 Binary files a/src/main/resources/runner-classes/eu/stamp_project/testrunner/runner/coverage/JUnit5JacocoRunnerPerTestMethod.class and b/src/main/resources/runner-classes/eu/stamp_project/testrunner/runner/coverage/JUnit5JacocoRunnerPerTestMethod.class differ diff --git a/src/main/resources/runner-classes/eu/stamp_project/testrunner/runner/coverage/JacocoRunner.class b/src/main/resources/runner-classes/eu/stamp_project/testrunner/runner/coverage/JacocoRunner.class index 19415a60..b01b09bf 100644 Binary files a/src/main/resources/runner-classes/eu/stamp_project/testrunner/runner/coverage/JacocoRunner.class and b/src/main/resources/runner-classes/eu/stamp_project/testrunner/runner/coverage/JacocoRunner.class differ diff --git a/src/main/resources/runner-classes/eu/stamp_project/testrunner/runner/coverage/JacocoRunnerPerTestMethod.class b/src/main/resources/runner-classes/eu/stamp_project/testrunner/runner/coverage/JacocoRunnerPerTestMethod.class index 57dc8bd3..966efa9c 100644 Binary files a/src/main/resources/runner-classes/eu/stamp_project/testrunner/runner/coverage/JacocoRunnerPerTestMethod.class and b/src/main/resources/runner-classes/eu/stamp_project/testrunner/runner/coverage/JacocoRunnerPerTestMethod.class differ diff --git a/src/main/resources/runner-classes/eu/stamp_project/testrunner/runner/coverage/MemoryClassLoader.class b/src/main/resources/runner-classes/eu/stamp_project/testrunner/runner/coverage/MemoryClassLoader.class index 168371f2..acba23cb 100644 Binary files a/src/main/resources/runner-classes/eu/stamp_project/testrunner/runner/coverage/MemoryClassLoader.class and b/src/main/resources/runner-classes/eu/stamp_project/testrunner/runner/coverage/MemoryClassLoader.class differ diff --git a/src/test/java/eu/stamp_project/testrunner/EntryPointJUnit5Test.java b/src/test/java/eu/stamp_project/testrunner/EntryPointJUnit5Test.java index bdd008ce..b308b1cf 100644 --- a/src/test/java/eu/stamp_project/testrunner/EntryPointJUnit5Test.java +++ b/src/test/java/eu/stamp_project/testrunner/EntryPointJUnit5Test.java @@ -2,9 +2,12 @@ import eu.stamp_project.testrunner.listener.Coverage; import eu.stamp_project.testrunner.listener.CoveragePerTestMethod; +import eu.stamp_project.testrunner.listener.CoveredTestResultPerTestMethod; import eu.stamp_project.testrunner.listener.TestResult; +import eu.stamp_project.testrunner.listener.impl.CoverageDetailed; import eu.stamp_project.testrunner.listener.pit.AbstractPitResult; import eu.stamp_project.testrunner.runner.Failure; +import eu.stamp_project.testrunner.runner.ParserOptions; import eu.stamp_project.testrunner.utils.ConstantsHelper; import org.junit.After; import org.junit.Before; @@ -33,6 +36,7 @@ public void setUp() throws Exception { EntryPoint.outPrintStream = null; EntryPoint.errPrintStream = null; EntryPoint.jUnit5Mode = true; + EntryPoint.coverageDetail = ParserOptions.CoverageTransformerDetail.SUMMARIZED; } @After @@ -398,4 +402,103 @@ Test the runCoveragePerTestMethods() of EntryPoint. assertEquals(23, coveragePerTestMethod.getCoverageOf("test8").getInstructionsCovered());// TODO something may be wrong here. The instruction coverage of test 8 is 23 assertEquals(EntryPointTest.NUMBER_OF_INSTRUCTIONS, coveragePerTestMethod.getCoverageOf("test8").getInstructionsTotal()); } + + @Test + public void testRunCoveredTestResultPerTestMethods() throws Exception { + + /* + Test the runCoveredTestResultPerTestMethods() of EntryPoint. + It should return the CoveredTestResultPerTestMethod with the instruction coverage computed by Jacoco. + */ + final String classpath = MAVEN_HOME + "org/jacoco/org.jacoco.core/0.7.9/org.jacoco.core-0.7.9.jar" + ConstantsHelper.PATH_SEPARATOR + + MAVEN_HOME + "org/ow2/asm/asm-debug-all/5.2/asm-debug-all-5.2.jar" + ConstantsHelper.PATH_SEPARATOR + + MAVEN_HOME + "commons-io/commons-io/2.5/commons-io-2.5.jar" + ConstantsHelper.PATH_SEPARATOR + + JUNIT_CP + ConstantsHelper.PATH_SEPARATOR + JUNIT5_CP; + + final CoveredTestResultPerTestMethod coveredTestResultPerTestMethod = EntryPoint.runCoveredTestResultPerTestMethods( + classpath + ConstantsHelper.PATH_SEPARATOR + TEST_PROJECT_CLASSES, + TEST_PROJECT_CLASSES, + "junit5.TestSuiteExample", + new String[]{"test8", "test3"} + ); + + // Assert test results + assertEquals(2, coveredTestResultPerTestMethod.getRunningTests().size()); + assertEquals(2, coveredTestResultPerTestMethod.getPassingTests().size()); + assertEquals(0, coveredTestResultPerTestMethod.getFailingTests().size()); + assertEquals(0, coveredTestResultPerTestMethod.getIgnoredTests().size()); + + // Assert coverage + assertEquals(2, coveredTestResultPerTestMethod.getCoverageResultsMap().size()); + assertEquals(23, coveredTestResultPerTestMethod.getCoverageOf("test3").getInstructionsCovered()); + assertEquals(EntryPointTest.NUMBER_OF_INSTRUCTIONS, coveredTestResultPerTestMethod.getCoverageOf("test3").getInstructionsTotal()); + assertEquals(23, coveredTestResultPerTestMethod.getCoverageOf("test8").getInstructionsCovered()); + assertEquals(EntryPointTest.NUMBER_OF_INSTRUCTIONS, coveredTestResultPerTestMethod.getCoverageOf("test8").getInstructionsTotal()); + } + + // FIXME: This test highlights a bug where same named test methods from different classes are mapped to the same key + // See: https://github.com/STAMP-project/test-runner/issues/86 + @Test + @Ignore + public void testRunCoveredTestResultPerTestMethodsSameNameMethods() throws Exception { + + /* + Test the runCoveredTestResultPerTestMethods() of EntryPoint. + It should return the CoveredTestResultPerTestMethod with the instruction coverage computed by Jacoco. + */ + final String classpath = MAVEN_HOME + "org/jacoco/org.jacoco.core/0.7.9/org.jacoco.core-0.7.9.jar" + ConstantsHelper.PATH_SEPARATOR + + MAVEN_HOME + "org/ow2/asm/asm-debug-all/5.2/asm-debug-all-5.2.jar" + ConstantsHelper.PATH_SEPARATOR + + MAVEN_HOME + "commons-io/commons-io/2.5/commons-io-2.5.jar" + ConstantsHelper.PATH_SEPARATOR + + JUNIT_CP + ConstantsHelper.PATH_SEPARATOR + JUNIT5_CP; + + final CoveredTestResultPerTestMethod coveredTestResultPerTestMethod = EntryPoint.runCoveredTestResultPerTestMethods( + classpath + ConstantsHelper.PATH_SEPARATOR + TEST_PROJECT_CLASSES, + TEST_PROJECT_CLASSES, + new String[]{"junit5.TestSuiteExample", "junit5.TestSuiteExample2"}, + new String[]{"junit5.TestSuiteExample#test3", "junit5.TestSuiteExample2#test3"} + ); + + System.out.println(coveredTestResultPerTestMethod); + } + + @Test + public void testRunCoveredTestResultPerTestMethodsDetailedCoverage() throws Exception { + EntryPoint.coverageDetail = ParserOptions.CoverageTransformerDetail.DETAIL; + + /* + Test the runCoveredTestResultPerTestMethods() of EntryPoint. + It should return the CoveredTestResultPerTestMethod with the instruction coverage computed by Jacoco. + */ + final String classpath = MAVEN_HOME + "org/jacoco/org.jacoco.core/0.7.9/org.jacoco.core-0.7.9.jar" + ConstantsHelper.PATH_SEPARATOR + + MAVEN_HOME + "org/ow2/asm/asm-debug-all/5.2/asm-debug-all-5.2.jar" + ConstantsHelper.PATH_SEPARATOR + + MAVEN_HOME + "commons-io/commons-io/2.5/commons-io-2.5.jar" + ConstantsHelper.PATH_SEPARATOR + + JUNIT_CP + ConstantsHelper.PATH_SEPARATOR + JUNIT5_CP; + + final CoveredTestResultPerTestMethod coveredTestResultPerTestMethod = EntryPoint.runCoveredTestResultPerTestMethods( + classpath + ConstantsHelper.PATH_SEPARATOR + TEST_PROJECT_CLASSES, + TEST_PROJECT_CLASSES, + "junit5.TestSuiteExample", + new String[]{"test8", "test3"} + ); + + // Assert test results + assertEquals(2, coveredTestResultPerTestMethod.getRunningTests().size()); + assertEquals(2, coveredTestResultPerTestMethod.getPassingTests().size()); + assertEquals(0, coveredTestResultPerTestMethod.getFailingTests().size()); + assertEquals(0, coveredTestResultPerTestMethod.getIgnoredTests().size()); + + // Assert detailed coverage + assertEquals(2, coveredTestResultPerTestMethod.getCoverageResultsMap().size()); + + assertTrue(coveredTestResultPerTestMethod.getCoverageOf("test3") instanceof CoverageDetailed); + CoverageDetailed coverageDetailed = (CoverageDetailed) coveredTestResultPerTestMethod.getCoverageOf("test3"); + assertNotNull(coverageDetailed.getDetailedCoverage()); + assertEquals(5, coverageDetailed.getDetailedCoverage().size()); + + assertTrue(coveredTestResultPerTestMethod.getCoverageOf("test8") instanceof CoverageDetailed); + coverageDetailed = (CoverageDetailed) coveredTestResultPerTestMethod.getCoverageOf("test8"); + assertNotNull(coverageDetailed.getDetailedCoverage()); + assertEquals(5, coverageDetailed.getDetailedCoverage().size()); + } + } diff --git a/src/test/java/eu/stamp_project/testrunner/EntryPointTest.java b/src/test/java/eu/stamp_project/testrunner/EntryPointTest.java index 1280392a..c41f990e 100644 --- a/src/test/java/eu/stamp_project/testrunner/EntryPointTest.java +++ b/src/test/java/eu/stamp_project/testrunner/EntryPointTest.java @@ -2,9 +2,13 @@ import eu.stamp_project.testrunner.listener.Coverage; import eu.stamp_project.testrunner.listener.CoveragePerTestMethod; +import eu.stamp_project.testrunner.listener.CoveredTestResultPerTestMethod; import eu.stamp_project.testrunner.listener.TestResult; +import eu.stamp_project.testrunner.listener.impl.CoverageDetailed; +import eu.stamp_project.testrunner.listener.impl.CoverageFromClass; import eu.stamp_project.testrunner.listener.pit.AbstractPitResult; import eu.stamp_project.testrunner.runner.Failure; +import eu.stamp_project.testrunner.runner.ParserOptions; import eu.stamp_project.testrunner.utils.ConstantsHelper; import org.junit.After; import org.junit.Before; @@ -39,6 +43,7 @@ public void setUp() { EntryPoint.errPrintStream = null; EntryPoint.verbose = true; EntryPoint.setMutationEngine(ConstantsHelper.MutationEngine.DESCARTES); + EntryPoint.coverageDetail = ParserOptions.CoverageTransformerDetail.SUMMARIZED; } @After @@ -452,4 +457,145 @@ public void testOnParametrizedForOneTestMethodCoveragePerTestMethod() throws Tim "test3:test4:test7" ); } + + @Test + public void testRunCoveredTestResultPerTestMethods() throws Exception { + + /* + Test the runCoveredTestResultPerTestMethods() of EntryPoint. + It should return the CoveredTestResult with the instruction coverage computed by Jacoco. + */ + final String classpath = MAVEN_HOME + "org/jacoco/org.jacoco.core/0.7.9/org.jacoco.core-0.7.9.jar" + ConstantsHelper.PATH_SEPARATOR + + MAVEN_HOME + "org/ow2/asm/asm-debug-all/5.2/asm-debug-all-5.2.jar" + ConstantsHelper.PATH_SEPARATOR + + MAVEN_HOME + "commons-io/commons-io/2.5/commons-io-2.5.jar" + ConstantsHelper.PATH_SEPARATOR + + JUNIT_CP + ConstantsHelper.PATH_SEPARATOR + JUNIT5_CP; + + final CoveredTestResultPerTestMethod coveredTestResultPerTestMethod = EntryPoint.runCoveredTestResultPerTestMethods( + classpath + ConstantsHelper.PATH_SEPARATOR + TEST_PROJECT_CLASSES, + TEST_PROJECT_CLASSES, + "example.TestSuiteExample", + new String[]{"test8", "test3"} + ); + + // Assert test results + assertEquals(2, coveredTestResultPerTestMethod.getRunningTests().size()); + assertEquals(2, coveredTestResultPerTestMethod.getPassingTests().size()); + assertEquals(0, coveredTestResultPerTestMethod.getFailingTests().size()); + assertEquals(0, coveredTestResultPerTestMethod.getIgnoredTests().size()); + + // Assert coverage + assertEquals(2, coveredTestResultPerTestMethod.getCoverageResultsMap().size()); + assertEquals(23, coveredTestResultPerTestMethod.getCoverageOf("test3").getInstructionsCovered()); + assertEquals(NUMBER_OF_INSTRUCTIONS, coveredTestResultPerTestMethod.getCoverageOf("test3").getInstructionsTotal()); + assertEquals(23, coveredTestResultPerTestMethod.getCoverageOf("test8").getInstructionsCovered()); + assertEquals(NUMBER_OF_INSTRUCTIONS, coveredTestResultPerTestMethod.getCoverageOf("test8").getInstructionsTotal()); + } + + @Ignore + @Test + public void testOnParametrizedForOneTestMethodCoveredTestResultPerTestMethod() throws TimeoutException { + + /* + Test the execution of Parametrized test + */ + + final CoveredTestResultPerTestMethod coveredTestResultPerTestMethod = EntryPoint.runCoveredTestResultPerTestMethods( + JUNIT_CP + ConstantsHelper.PATH_SEPARATOR + TEST_PROJECT_CLASSES + + ConstantsHelper.PATH_SEPARATOR + JUNIT5_CP, + TEST_PROJECT_CLASSES, + "example.ParametrizedTestSuiteExample", + "test3:test4:test7" + ); + + // Assert test results + // FIXME: Assertion failing because there the keys for the original test method names are missing + assertEquals(9, coveredTestResultPerTestMethod.getRunningTests().size()); + assertEquals(9, coveredTestResultPerTestMethod.getPassingTests().size()); + assertEquals(0, coveredTestResultPerTestMethod.getFailingTests().size()); + assertEquals(0, coveredTestResultPerTestMethod.getIgnoredTests().size()); + + // Assert coverage + assertEquals(9, coveredTestResultPerTestMethod.getCoverageResultsMap().size()); + assertEquals(23, coveredTestResultPerTestMethod.getCoverageOf("test3[0]").getInstructionsCovered()); + assertEquals(NUMBER_OF_INSTRUCTIONS, coveredTestResultPerTestMethod.getCoverageOf("test3[0]").getInstructionsTotal()); + assertEquals(23, coveredTestResultPerTestMethod.getCoverageOf("test3[1]").getInstructionsCovered()); + assertEquals(NUMBER_OF_INSTRUCTIONS, coveredTestResultPerTestMethod.getCoverageOf("test3[1]").getInstructionsTotal()); + + assertEquals(26, coveredTestResultPerTestMethod.getCoverageOf("test4[0]").getInstructionsCovered()); + assertEquals(NUMBER_OF_INSTRUCTIONS, coveredTestResultPerTestMethod.getCoverageOf("test4[0]").getInstructionsTotal()); + assertEquals(26, coveredTestResultPerTestMethod.getCoverageOf("test4[1]").getInstructionsCovered()); + assertEquals(NUMBER_OF_INSTRUCTIONS, coveredTestResultPerTestMethod.getCoverageOf("test4[1]").getInstructionsTotal()); + + + assertEquals(23, coveredTestResultPerTestMethod.getCoverageOf("test7[0]").getInstructionsCovered()); + assertEquals(NUMBER_OF_INSTRUCTIONS, coveredTestResultPerTestMethod.getCoverageOf("test7[0]").getInstructionsTotal()); + assertEquals(23, coveredTestResultPerTestMethod.getCoverageOf("test7[1]").getInstructionsCovered()); + assertEquals(NUMBER_OF_INSTRUCTIONS, coveredTestResultPerTestMethod.getCoverageOf("test7[1]").getInstructionsTotal()); + } + + // FIXME: This test highlights a bug where same named test methods from different classes are mapped to the same key + // See: https://github.com/STAMP-project/test-runner/issues/86 + @Ignore + @Test + public void testRunCoveredTestResultPerTestMethodsSameNamedMethods() throws Exception { + + /* + Test the runCoveredTestResultPerTestMethods() of EntryPoint. + It should return the CoveredTestResult with the instruction coverage computed by Jacoco. + */ + final String classpath = MAVEN_HOME + "org/jacoco/org.jacoco.core/0.7.9/org.jacoco.core-0.7.9.jar" + ConstantsHelper.PATH_SEPARATOR + + MAVEN_HOME + "org/ow2/asm/asm-debug-all/5.2/asm-debug-all-5.2.jar" + ConstantsHelper.PATH_SEPARATOR + + MAVEN_HOME + "commons-io/commons-io/2.5/commons-io-2.5.jar" + ConstantsHelper.PATH_SEPARATOR + + JUNIT_CP + ConstantsHelper.PATH_SEPARATOR + JUNIT5_CP; + + final CoveredTestResultPerTestMethod coveredTestResultPerTestMethod = EntryPoint.runCoveredTestResultPerTestMethods( + classpath + ConstantsHelper.PATH_SEPARATOR + TEST_PROJECT_CLASSES, + TEST_PROJECT_CLASSES, + new String[]{"example.TestSuiteExample", "example.TestSuiteExample2"}, + new String[]{"example.TestSuiteExample#test3", "example.TestSuiteExample2#test3"} + ); + + System.out.println(coveredTestResultPerTestMethod); + } + + @Test + public void testRunCoveredTestResultPerTestMethodsDetailedCoverage() throws Exception { + EntryPoint.coverageDetail = ParserOptions.CoverageTransformerDetail.DETAIL; + + /* + Test the runCoveredTestResultPerTestMethods() of EntryPoint. + It should return the CoveredTestResultPerTestMethod with the instruction coverage computed by Jacoco. + */ + final String classpath = MAVEN_HOME + "org/jacoco/org.jacoco.core/0.7.9/org.jacoco.core-0.7.9.jar" + ConstantsHelper.PATH_SEPARATOR + + MAVEN_HOME + "org/ow2/asm/asm-debug-all/5.2/asm-debug-all-5.2.jar" + ConstantsHelper.PATH_SEPARATOR + + MAVEN_HOME + "commons-io/commons-io/2.5/commons-io-2.5.jar" + ConstantsHelper.PATH_SEPARATOR + + JUNIT_CP + ConstantsHelper.PATH_SEPARATOR + JUNIT5_CP; + + final CoveredTestResultPerTestMethod coveredTestResultPerTestMethod = EntryPoint.runCoveredTestResultPerTestMethods( + classpath + ConstantsHelper.PATH_SEPARATOR + TEST_PROJECT_CLASSES, + TEST_PROJECT_CLASSES, + "example.TestSuiteExample", + new String[]{"test8", "test3"} + ); + + // Assert test results + assertEquals(2, coveredTestResultPerTestMethod.getRunningTests().size()); + assertEquals(2, coveredTestResultPerTestMethod.getPassingTests().size()); + assertEquals(0, coveredTestResultPerTestMethod.getFailingTests().size()); + assertEquals(0, coveredTestResultPerTestMethod.getIgnoredTests().size()); + + // Assert detailed coverage + assertEquals(2, coveredTestResultPerTestMethod.getCoverageResultsMap().size()); + + assertTrue(coveredTestResultPerTestMethod.getCoverageOf("test3") instanceof CoverageDetailed); + CoverageDetailed coverageDetailed = (CoverageDetailed) coveredTestResultPerTestMethod.getCoverageOf("test3"); + assertNotNull(coverageDetailed.getDetailedCoverage()); + assertEquals(5, coverageDetailed.getDetailedCoverage().size()); + + assertTrue(coveredTestResultPerTestMethod.getCoverageOf("test8") instanceof CoverageDetailed); + coverageDetailed = (CoverageDetailed) coveredTestResultPerTestMethod.getCoverageOf("test8"); + assertNotNull(coverageDetailed.getDetailedCoverage()); + assertEquals(5, coverageDetailed.getDetailedCoverage().size()); + } + } diff --git a/src/test/java/eu/stamp_project/testrunner/runner/coverage/JUnit4JacocoRunnerCoveredResultPerTestMethodTest.java b/src/test/java/eu/stamp_project/testrunner/runner/coverage/JUnit4JacocoRunnerCoveredResultPerTestMethodTest.java new file mode 100644 index 00000000..074e17cb --- /dev/null +++ b/src/test/java/eu/stamp_project/testrunner/runner/coverage/JUnit4JacocoRunnerCoveredResultPerTestMethodTest.java @@ -0,0 +1,127 @@ +package eu.stamp_project.testrunner.runner.coverage; + +import eu.stamp_project.testrunner.AbstractTest; +import eu.stamp_project.testrunner.listener.CoveredTestResultPerTestMethod; +import eu.stamp_project.testrunner.listener.impl.CoveredTestResultPerTestMethodImpl; +import eu.stamp_project.testrunner.runner.ParserOptions; +import org.junit.Ignore; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class JUnit4JacocoRunnerCoveredResultPerTestMethodTest extends AbstractTest { + + // FIXME: The behaviour in this case is not consistent + // There should be an equal number of keys in both running tests and the coverage map + // There should also always be a key for the method given as input + @Test + @Ignore + public void testWithoutNewJvmOnTestClassParametrized() throws Exception { + + /* + Using the api to compute the coverage on a test class + */ + + JUnit4JacocoRunnerCoveredResultPerTestMethod.main(new String[]{ + ParserOptions.FLAG_pathToCompiledClassesOfTheProject, TEST_PROJECT_CLASSES, + ParserOptions.FLAG_fullQualifiedNameOfTestClassToRun, "example.ParametrizedTest", + ParserOptions.FLAG_testMethodNamesToRun, "test" + } + ); + final CoveredTestResultPerTestMethod load = CoveredTestResultPerTestMethodImpl.load(); + System.out.println(load); + + assertEquals(5, load.getRunningTests().size()); + assertEquals(5, load.getPassingTests().size()); + assertEquals(0, load.getFailingTests().size()); + assertEquals(0, load.getIgnoredTests().size()); + + assertEquals(5, load.getCoverageResultsMap().size()); + assertEquals(34, load.getCoverageResultsMap().get("test").getInstructionsCovered()); + assertEquals(23, load.getCoverageResultsMap().get("test[0]").getInstructionsCovered()); + assertEquals(26, load.getCoverageResultsMap().get("test[1]").getInstructionsCovered()); + assertEquals(19, load.getCoverageResultsMap().get("test[2]").getInstructionsCovered()); + assertEquals(19, load.getCoverageResultsMap().get("test[3]").getInstructionsCovered()); + + System.out.println(load.getCoverageResultsMap().get("test").getExecutionPath()); + } + + @Test + public void testWithoutNewJvmOnTestCases() throws Exception { + + /* + Using the api to compute the coverage on test cases + */ + + JUnit4JacocoRunnerCoveredResultPerTestMethod.main(new String[]{ + ParserOptions.FLAG_pathToCompiledClassesOfTheProject, TEST_PROJECT_CLASSES, + ParserOptions.FLAG_fullQualifiedNameOfTestClassToRun, "example.TestSuiteExample", + ParserOptions.FLAG_testMethodNamesToRun, "test3:test2:copyOftest2" + } + ); + final CoveredTestResultPerTestMethod load = CoveredTestResultPerTestMethodImpl.load(); + System.out.println(load); + + assertEquals(3, load.getRunningTests().size()); + assertEquals(3, load.getPassingTests().size()); + assertEquals(0, load.getFailingTests().size()); + assertEquals(0, load.getIgnoredTests().size()); + + assertEquals(3, load.getCoverageResultsMap().size()); + assertEquals(23, load.getCoverageResultsMap().get("test2").getInstructionsCovered()); + assertEquals(23, load.getCoverageResultsMap().get("test3").getInstructionsCovered()); + assertEquals(23, load.getCoverageResultsMap().get("copyOftest2").getInstructionsCovered()); + System.out.println(load.getCoverageResultsMap().get("test2").getExecutionPath()); + System.out.println(load.getCoverageResultsMap().get("copyOftest2").getExecutionPath()); + System.out.println(load.getCoverageResultsMap().get("test3").getExecutionPath()); + } + + @Test + public void testWithoutNewJvmOnTestClassAll() throws Exception { + + /* + Using the api to compute the coverage on test cases + */ + + JUnit4JacocoRunnerCoveredResultPerTestMethod.main(new String[]{ + ParserOptions.FLAG_pathToCompiledClassesOfTheProject, TEST_PROJECT_CLASSES, + ParserOptions.FLAG_fullQualifiedNameOfTestClassToRun, "example.TestSuiteExample", + } + ); + final CoveredTestResultPerTestMethod load = CoveredTestResultPerTestMethodImpl.load(); + + System.out.println(load); + load.getCoverageResultsMap() + .keySet() + .stream() + .map(s -> s + " -> " + load.getCoverageResultsMap().get(s).getExecutionPath()) + .forEach(System.out::println); + for (String expectedExecutionPath : expectedExecutionPathsTest3) { + assertEquals(23, load.getCoverageResultsMap().get("test3").getInstructionsCovered()); + assertTrue(load.getCoverageResultsMap().get("test3").getExecutionPath().contains(expectedExecutionPath)); + } + for (String expectedExecutionPath : expectedExecutionPathsTest4) { + assertEquals(26, load.getCoverageResultsMap().get("test4").getInstructionsCovered()); + assertTrue(load.getCoverageResultsMap().get("test4").getExecutionPath().contains(expectedExecutionPath)); + } + } + + private static final String[] expectedExecutionPathsTest3 = { + "tobemocked/LoginDao:0,0", + "tobemocked/LoginController:0,0,0,0,0,0,0,0,0,0,0,0,0,0,0", + "tobemocked/LoginService:0,0,0,0,0,0,0,0,0,0,0,0,0,0", + "example/Example:2,0,0,4,4,0,0,2,0,2,5,1,0,3", + "tobemocked/UserForm:0,0" + }; + + private static final String[] expectedExecutionPathsTest4 = { + "tobemocked/LoginDao:0,0", + "tobemocked/LoginController:0,0,0,0,0,0,0,0,0,0,0,0,0,0,0", + "tobemocked/LoginService:0,0,0,0,0,0,0,0,0,0,0,0,0,0", + "example/Example:2,0,0,4,0,0,7,2,0,2,5,1,0,3", + "tobemocked/UserForm:0,0" + }; + + +} \ No newline at end of file diff --git a/src/test/java/eu/stamp_project/testrunner/runner/coverage/JUnit5JacocoRunnerCoveredResultPerTestMethodTest.java b/src/test/java/eu/stamp_project/testrunner/runner/coverage/JUnit5JacocoRunnerCoveredResultPerTestMethodTest.java new file mode 100644 index 00000000..752bdb80 --- /dev/null +++ b/src/test/java/eu/stamp_project/testrunner/runner/coverage/JUnit5JacocoRunnerCoveredResultPerTestMethodTest.java @@ -0,0 +1,110 @@ +package eu.stamp_project.testrunner.runner.coverage; + +import eu.stamp_project.testrunner.AbstractTest; +import eu.stamp_project.testrunner.listener.CoveredTestResultPerTestMethod; +import eu.stamp_project.testrunner.listener.impl.CoveredTestResultPerTestMethodImpl; +import eu.stamp_project.testrunner.runner.ParserOptions; +import org.junit.Ignore; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class JUnit5JacocoRunnerCoveredResultPerTestMethodTest extends AbstractTest { + + // FIXME: Parametrized tests are not yet supported for JUnit5 + @Ignore + @Test + public void testWithoutNewJvmOnTestClassParametrized() throws Exception { + + /* + Using the api to compute the coverage on a test class + */ + + JUnit5JacocoRunnerCoveredResultPerTestMethod.main(new String[]{ + ParserOptions.FLAG_pathToCompiledClassesOfTheProject, TEST_PROJECT_CLASSES, + ParserOptions.FLAG_fullQualifiedNameOfTestClassToRun, "junit5.ParametrizedTest", + ParserOptions.FLAG_testMethodNamesToRun, "test" + } + ); + final CoveredTestResultPerTestMethod load = CoveredTestResultPerTestMethodImpl.load(); + System.out.println(load); + assertEquals(34, load.getCoverageResultsMap().get("test").getInstructionsCovered()); + System.out.println(load.getCoverageResultsMap().get("test").getExecutionPath()); + } + + @Test + public void testWithoutNewJvmOnTestCases() throws Exception { + + /* + Using the api to compute the coverage on test cases + */ + + JUnit5JacocoRunnerCoveredResultPerTestMethod.main(new String[]{ + ParserOptions.FLAG_pathToCompiledClassesOfTheProject, TEST_PROJECT_CLASSES, + ParserOptions.FLAG_fullQualifiedNameOfTestClassToRun, "junit5.TestSuiteExample", + ParserOptions.FLAG_testMethodNamesToRun, "test3:test2" + } + ); + final CoveredTestResultPerTestMethod load = CoveredTestResultPerTestMethodImpl.load(); + System.out.println(load); + + assertEquals(2, load.getRunningTests().size()); + assertEquals(2, load.getPassingTests().size()); + assertEquals(0, load.getFailingTests().size()); + assertEquals(0, load.getIgnoredTests().size()); + + assertEquals(2, load.getCoverageResultsMap().size()); + assertEquals(23, load.getCoverageResultsMap().get("test2").getInstructionsCovered()); + assertEquals(23, load.getCoverageResultsMap().get("test3").getInstructionsCovered()); + System.out.println(load.getCoverageResultsMap().get("test2").getExecutionPath()); + System.out.println(load.getCoverageResultsMap().get("test3").getExecutionPath()); + } + + @Test + public void testWithoutNewJvmOnTestClassAll() throws Exception { + + /* + Using the api to compute the coverage on test cases + */ + + JUnit5JacocoRunnerCoveredResultPerTestMethod.main(new String[]{ + ParserOptions.FLAG_pathToCompiledClassesOfTheProject, TEST_PROJECT_CLASSES, + ParserOptions.FLAG_fullQualifiedNameOfTestClassToRun, "junit5.TestSuiteExample", + } + ); + final CoveredTestResultPerTestMethod load = CoveredTestResultPerTestMethodImpl.load(); + System.out.println(load); + load.getCoverageResultsMap() + .keySet() + .stream() + .map(s -> s + " -> " + load.getCoverageResultsMap().get(s).getExecutionPath()) + .forEach(System.out::println); + for (String expectedExecutionPath : expectedExecutionPathsTest3) { + assertEquals(23, load.getCoverageResultsMap().get("test3").getInstructionsCovered()); + assertTrue(load.getCoverageResultsMap().get("test3").getExecutionPath().contains(expectedExecutionPath)); + } + for (String expectedExecutionPath : expectedExecutionPathsTest4) { + assertEquals(26, load.getCoverageResultsMap().get("test4").getInstructionsCovered()); + assertTrue(load.getCoverageResultsMap().get("test4").getExecutionPath().contains(expectedExecutionPath)); + } + } + + private static final String[] expectedExecutionPathsTest3 = { + "tobemocked/LoginDao:0,0", + "tobemocked/LoginController:0,0,0,0,0,0,0,0,0,0,0,0,0,0,0", + "tobemocked/LoginService:0,0,0,0,0,0,0,0,0,0,0,0,0,0", + "example/Example:2,0,0,4,4,0,0,2,0,2,5,1,0,3", + "tobemocked/UserForm:0,0" + }; + + private static final String[] expectedExecutionPathsTest4 = { + "tobemocked/LoginDao:0,0", + "tobemocked/LoginController:0,0,0,0,0,0,0,0,0,0,0,0,0,0,0", + "tobemocked/LoginService:0,0,0,0,0,0,0,0,0,0,0,0,0,0", + "example/Example:2,0,0,4,0,0,7,2,0,2,5,1,0,3", + "tobemocked/UserForm:0,0" + }; + + +}