From 5e9f11d3df3c1fdd4beea1014ca5fae990588d23 Mon Sep 17 00:00:00 2001 From: Martin Armbruster Date: Wed, 26 Jul 2023 13:50:11 +0200 Subject: [PATCH 01/10] Added new dependencies for storage measurements (#23). --- jamopp.tests/pom.xml | 8 ++++++++ pom.xml | 10 ++++++++++ 2 files changed, 18 insertions(+) diff --git a/jamopp.tests/pom.xml b/jamopp.tests/pom.xml index 5610d278..67676e8f 100644 --- a/jamopp.tests/pom.xml +++ b/jamopp.tests/pom.xml @@ -123,5 +123,13 @@ org.apache.commons commons-compress + + commons-io + commons-io + + + org.eclipse.emfcloud + emfjson-jackson + diff --git a/pom.xml b/pom.xml index 011e4ef9..0358f1ef 100644 --- a/pom.xml +++ b/pom.xml @@ -148,6 +148,16 @@ commons-compress 1.23.0 + + commons-io + commons-io + 2.13.0 + + + org.eclipse.emfcloud + emfjson-jackson + 2.2.0 + From 3c86d0d2f732729d806dc33e2c5176f3b07cc0a5 Mon Sep 17 00:00:00 2001 From: Martin Armbruster Date: Wed, 26 Jul 2023 14:35:10 +0200 Subject: [PATCH 02/10] Extended the performance data with storage data (#23). --- .../test/performance/PerformanceData.java | 24 +++++-- .../test/performance/StoragePerformance.java | 62 +++++++++++++++++++ 2 files changed, 82 insertions(+), 4 deletions(-) create mode 100644 jamopp.tests/src/test/java/tools/mdsd/jamopp/test/performance/StoragePerformance.java diff --git a/jamopp.tests/src/test/java/tools/mdsd/jamopp/test/performance/PerformanceData.java b/jamopp.tests/src/test/java/tools/mdsd/jamopp/test/performance/PerformanceData.java index 361b382f..af78d51d 100644 --- a/jamopp.tests/src/test/java/tools/mdsd/jamopp/test/performance/PerformanceData.java +++ b/jamopp.tests/src/test/java/tools/mdsd/jamopp/test/performance/PerformanceData.java @@ -19,6 +19,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; +import java.util.List; import com.google.gson.Gson; @@ -27,13 +28,28 @@ */ public class PerformanceData { private ArrayList points = new ArrayList<>(); + private ArrayList storage = new ArrayList<>(); - public ArrayList getPoints() { - return points; + public List getPoints() { + return (List) points.clone(); + } + + public void addPoint(PerformanceDataPoint newPoint) { + this.points.add(newPoint); } - public void setPoints(ArrayList points) { - this.points = points; + public void setPoints(List points) { + this.points.clear(); + this.points.addAll(points); + } + + public List getStorage() { + return (List) storage.clone(); + } + + public void setStorage(List storage) { + this.storage.clear(); + this.storage.addAll(storage); } public double getAverageParseTime() { diff --git a/jamopp.tests/src/test/java/tools/mdsd/jamopp/test/performance/StoragePerformance.java b/jamopp.tests/src/test/java/tools/mdsd/jamopp/test/performance/StoragePerformance.java new file mode 100644 index 00000000..8098e63c --- /dev/null +++ b/jamopp.tests/src/test/java/tools/mdsd/jamopp/test/performance/StoragePerformance.java @@ -0,0 +1,62 @@ +/******************************************************************************* + * Copyright (c) 2023, Martin Armbruster + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Martin Armbruster + * - Initial implementation + ******************************************************************************/ + +package tools.mdsd.jamopp.test.performance; + +public class StoragePerformance { + private String id; + private long takenStorageByCodeFiles; + private long takenStorage; + private long overallFiles; + private long codeFiles; + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public long getTakenStorageByCodeFiles() { + return takenStorageByCodeFiles; + } + + public void setTakenStorageByCodeFiles(long takenStorageByCodeFiles) { + this.takenStorageByCodeFiles = takenStorageByCodeFiles; + } + + public long getTakenStorage() { + return takenStorage; + } + + public void setTakenStorage(long takenStorage) { + this.takenStorage = takenStorage; + } + + public long getOverallFiles() { + return overallFiles; + } + + public void setOverallFiles(long overallFiles) { + this.overallFiles = overallFiles; + } + + public long getCodeFiles() { + return codeFiles; + } + + public void setCodeFiles(long codeFiles) { + this.codeFiles = codeFiles; + } +} From 2ed8d2bce227658489b9ce12100c7fb09c059a91 Mon Sep 17 00:00:00 2001 From: Martin Armbruster Date: Thu, 27 Jul 2023 21:47:16 +0200 Subject: [PATCH 03/10] Extracted and generalized the output of Java models in the XMI format (#23). --- .../tools/mdsd/jamopp/test/OutputUtility.java | 83 +++++++++++++++++++ .../test/xmi/JavaXMISerializationTest.java | 59 +------------ 2 files changed, 85 insertions(+), 57 deletions(-) create mode 100644 jamopp.tests/src/test/java/tools/mdsd/jamopp/test/OutputUtility.java diff --git a/jamopp.tests/src/test/java/tools/mdsd/jamopp/test/OutputUtility.java b/jamopp.tests/src/test/java/tools/mdsd/jamopp/test/OutputUtility.java new file mode 100644 index 00000000..6719a0e4 --- /dev/null +++ b/jamopp.tests/src/test/java/tools/mdsd/jamopp/test/OutputUtility.java @@ -0,0 +1,83 @@ +package tools.mdsd.jamopp.test; + +import static org.junit.jupiter.api.Assertions.fail; + +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; + +import org.eclipse.emf.common.util.URI; +import org.eclipse.emf.ecore.resource.Resource; +import org.eclipse.emf.ecore.resource.ResourceSet; +import org.eclipse.emf.ecore.resource.impl.ResourceSetImpl; +import org.eclipse.emf.ecore.xmi.XMIResource; + +import tools.mdsd.jamopp.model.java.containers.CompilationUnit; +import tools.mdsd.jamopp.model.java.containers.JavaRoot; +import tools.mdsd.jamopp.model.java.containers.Package; + +public class OutputUtility { + public record TransferResult(ResourceSet targetSet, Map sourceTargetMapping) {}; + + public static TransferResult transferToOutput(ResourceSet sourceSet, String outputFolder, String fileExtension, boolean includeAllResources) { + int emptyFileName = 0; + + ResourceSet targetSet = new ResourceSetImpl(); + HashMap srcTrgMap = new HashMap<>(); + + for (Resource javaResource : new ArrayList<>(sourceSet.getResources())) { + if (javaResource.getContents().isEmpty()) { + System.out.println("WARNING: Emtpy Resource: " + javaResource.getURI()); + continue; + } + if (!includeAllResources && !javaResource.getURI().isFile()) { + continue; + } + + JavaRoot root = (JavaRoot) javaResource.getContents().get(0); + String outputFileName = "ERROR"; + if (root instanceof CompilationUnit cu) { + outputFileName = cu.getNamespacesAsString().replace(".", File.separator) + File.separator; + if (cu.getClassifiers().size() > 0) { + outputFileName += cu.getClassifiers().get(0).getName(); + } else { + outputFileName += emptyFileName++; + } + } else if (root instanceof Package) { + outputFileName = root.getNamespacesAsString() + .replace(".", File.separator) + File.separator + "package-info"; + if (outputFileName.startsWith(File.separator)) { + outputFileName = outputFileName.substring(1); + } + } else if (root instanceof tools.mdsd.jamopp.model.java.containers.Module) { + outputFileName = root.getNamespacesAsString() + .replace(".", File.separator) + File.separator + "module-info"; + } else { + fail(); + } + + File outputFile = new File("." + File.separator + outputFolder + + File.separator + outputFileName); + URI fileURI = URI.createFileURI(outputFile.getAbsolutePath()).appendFileExtension(fileExtension); + + Resource targetResource = targetSet.createResource(fileURI); + if (targetResource instanceof XMIResource xmiResource) { + xmiResource.setEncoding(StandardCharsets.UTF_8.toString()); + } + targetResource.getContents().addAll(javaResource.getContents()); + srcTrgMap.put(javaResource, targetResource); + } + + for (Resource targetResource : targetSet.getResources()) { + try { + targetResource.save(targetSet.getLoadOptions()); + } catch (Exception e) { + e.printStackTrace(); + } + } + + return new TransferResult(targetSet, srcTrgMap); + } +} diff --git a/jamopp.tests/src/test/java/tools/mdsd/jamopp/test/xmi/JavaXMISerializationTest.java b/jamopp.tests/src/test/java/tools/mdsd/jamopp/test/xmi/JavaXMISerializationTest.java index 324687d4..ba44bb19 100644 --- a/jamopp.tests/src/test/java/tools/mdsd/jamopp/test/xmi/JavaXMISerializationTest.java +++ b/jamopp.tests/src/test/java/tools/mdsd/jamopp/test/xmi/JavaXMISerializationTest.java @@ -21,30 +21,24 @@ import static org.junit.jupiter.api.Assertions.fail; import java.io.File; -import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; -import org.eclipse.emf.common.util.URI; import org.eclipse.emf.ecore.EObject; import org.eclipse.emf.ecore.EStructuralFeature; import org.eclipse.emf.ecore.resource.Resource; import org.eclipse.emf.ecore.resource.Resource.Diagnostic; import org.eclipse.emf.ecore.resource.ResourceSet; -import org.eclipse.emf.ecore.resource.impl.ResourceSetImpl; import org.eclipse.emf.ecore.util.EcoreUtil; import org.eclipse.emf.ecore.util.EcoreUtil.EqualityHelper; -import org.eclipse.emf.ecore.xmi.XMIResource; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; -import tools.mdsd.jamopp.model.java.containers.CompilationUnit; -import tools.mdsd.jamopp.model.java.containers.JavaRoot; -import tools.mdsd.jamopp.model.java.containers.Package; import tools.mdsd.jamopp.options.ParserOptions; import tools.mdsd.jamopp.parser.jdt.singlefile.JaMoPPJDTSingleFileParser; import tools.mdsd.jamopp.test.AbstractJaMoPPTests; +import tools.mdsd.jamopp.test.OutputUtility; public class JavaXMISerializationTest extends AbstractJaMoPPTests { @@ -88,56 +82,7 @@ public void testXMISerialization() throws Exception { } protected ResourceSet transferToXMI(ResourceSet sourceSet, boolean includeAllResources) throws Exception { - int emptyFileName = 0; - - ResourceSet targetSet = new ResourceSetImpl(); - - for (Resource javaResource : new ArrayList<>(sourceSet.getResources())) { - if (javaResource.getContents().isEmpty()) { - System.out.println("WARNING: Emtpy Resource: " + javaResource.getURI()); - continue; - } - if (!includeAllResources && !javaResource.getURI().isFile()) { - continue; - } - JavaRoot root = (JavaRoot) javaResource.getContents().get(0); - String outputFileName = "ERROR"; - if (root instanceof CompilationUnit) { - outputFileName = root.getNamespacesAsString().replace(".", File.separator) + File.separator; - CompilationUnit cu = (CompilationUnit) root; - if (cu.getClassifiers().size() > 0) { - outputFileName += cu.getClassifiers().get(0).getName(); - } else { - outputFileName += emptyFileName++; - } - - } else if (root instanceof Package) { - outputFileName = root.getNamespacesAsString() - .replace(".", File.separator) + File.separator + "package-info"; - if (outputFileName.startsWith(File.separator)) { - outputFileName = outputFileName.substring(1); - } - } else if (root instanceof tools.mdsd.jamopp.model.java.containers.Module) { - outputFileName = root.getNamespacesAsString() - .replace(".", File.separator) + File.separator + "module-info"; - } else { - fail(); - } - File outputFile = new File("." + File.separator + TEST_OUTPUT_FOLDER_NAME - + File.separator + outputFileName); - URI xmiFileURI = URI.createFileURI(outputFile.getAbsolutePath()).appendFileExtension("xmi"); - XMIResource xmiResource = (XMIResource) targetSet.createResource(xmiFileURI); - xmiResource.setEncoding(StandardCharsets.UTF_8.toString()); - xmiResource.getContents().addAll(javaResource.getContents()); - } - for (Resource xmiResource : targetSet.getResources()) { - try { - xmiResource.save(targetSet.getLoadOptions()); - } catch (Exception e) { - e.printStackTrace(); - } - } - return targetSet; + return OutputUtility.transferToOutput(sourceSet, TEST_OUTPUT_FOLDER_NAME, "xmi", includeAllResources).targetSet(); } protected void compare(ResourceSet rs) throws Exception { From f477804a15197edb7c658dbc6d9fa0cd3bc98f5a Mon Sep 17 00:00:00 2001 From: Martin Armbruster Date: Sun, 30 Jul 2023 14:14:46 +0200 Subject: [PATCH 04/10] Extended the performance test to also calculate the taken storage for different output formats (#23). --- .../tools/mdsd/jamopp/test/OutputUtility.java | 2 +- .../test/performance/PerformanceTest.java | 122 ++++++++++++++++-- 2 files changed, 109 insertions(+), 15 deletions(-) diff --git a/jamopp.tests/src/test/java/tools/mdsd/jamopp/test/OutputUtility.java b/jamopp.tests/src/test/java/tools/mdsd/jamopp/test/OutputUtility.java index 6719a0e4..e13569df 100644 --- a/jamopp.tests/src/test/java/tools/mdsd/jamopp/test/OutputUtility.java +++ b/jamopp.tests/src/test/java/tools/mdsd/jamopp/test/OutputUtility.java @@ -70,7 +70,7 @@ public static TransferResult transferToOutput(ResourceSet sourceSet, String outp srcTrgMap.put(javaResource, targetResource); } - for (Resource targetResource : targetSet.getResources()) { + for (Resource targetResource : new ArrayList<>(targetSet.getResources())) { try { targetResource.save(targetSet.getLoadOptions()); } catch (Exception e) { diff --git a/jamopp.tests/src/test/java/tools/mdsd/jamopp/test/performance/PerformanceTest.java b/jamopp.tests/src/test/java/tools/mdsd/jamopp/test/performance/PerformanceTest.java index 62806485..aa91f6c9 100644 --- a/jamopp.tests/src/test/java/tools/mdsd/jamopp/test/performance/PerformanceTest.java +++ b/jamopp.tests/src/test/java/tools/mdsd/jamopp/test/performance/PerformanceTest.java @@ -21,12 +21,19 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.HashSet; +import java.util.List; import java.util.Set; + import org.apache.logging.log4j.Logger; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.file.PathUtils; import org.apache.logging.log4j.LogManager; +import org.eclipse.emf.common.util.URI; import org.eclipse.emf.ecore.resource.Resource; import org.eclipse.emf.ecore.resource.ResourceSet; import org.eclipse.emf.ecore.util.EcoreUtil; +import org.eclipse.emfcloud.jackson.resource.JsonResourceFactory; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; @@ -34,6 +41,8 @@ import tools.mdsd.jamopp.parser.jdt.singlefile.JaMoPPJDTSingleFileParser; import tools.mdsd.jamopp.resource.JavaResource2; import tools.mdsd.jamopp.test.AbstractJaMoPPTests; +import tools.mdsd.jamopp.test.OutputUtility; +import tools.mdsd.jamopp.test.OutputUtility.TransferResult; import tools.mdsd.jamopp.test.bulk.SingleFileParserBulkTests; /** @@ -44,7 +53,27 @@ public class PerformanceTest extends AbstractJaMoPPTests { private static final Logger LOGGER = LogManager.getLogger("jamopp." + SingleFileParserBulkTests.class.getSimpleName()); private final String inputFolder = "target" + File.separator + "src-bulk" + File.separator + "TeaStore"; - private final Path parentOutput = Paths.get("output_performance"); + private final Path parentOutput = Paths.get("target", "tests", "output_performance"); + private final Path javaOutput = parentOutput.resolve("java"); + private final Path xmiOutput = parentOutput.resolve("xmi"); + private final Path jsonOutput = parentOutput.resolve("json"); + + @BeforeEach + public void setup() throws IOException { + super.initResourceFactory(); + Resource.Factory.Registry.INSTANCE.getExtensionToFactoryMap().put("json", new JsonResourceFactory()); + if (Files.exists(javaOutput)) { + PathUtils.deleteDirectory(javaOutput); + PathUtils.deleteDirectory(xmiOutput); + PathUtils.deleteDirectory(jsonOutput); + } + try { + Files.createDirectories(javaOutput); + Files.createDirectories(xmiOutput); + Files.createDirectories(jsonOutput); + } catch (IOException e1) { + } + } @Test public void measureTeaStoreFullResolution() { @@ -67,7 +96,7 @@ public void measureTeaStoreWithoutResolvingEverything() { ParserOptions.RESOLVE_BINDINGS_OF_INFERABLE_TYPES.setValue(Boolean.TRUE); ParserOptions.RESOLVE_EVERYTHING.setValue(Boolean.FALSE); ParserOptions.RESOLVE_ALL_BINDINGS.setValue(Boolean.TRUE); - measurePerformance("teastore-without-resolving-everything", 20, true); + measurePerformance("teastore-without-resolving-everything", 100, true); } @Test @@ -79,7 +108,7 @@ public void measureTeaStoreWithOneLevelResolution() { ParserOptions.RESOLVE_BINDINGS_OF_INFERABLE_TYPES.setValue(Boolean.TRUE); ParserOptions.RESOLVE_EVERYTHING.setValue(Boolean.FALSE); ParserOptions.RESOLVE_ALL_BINDINGS.setValue(Boolean.FALSE); - measurePerformance("teastore-one-level-resolution", 20, false); + measurePerformance("teastore-one-level-resolution", 100, false); } @Test @@ -91,7 +120,7 @@ public void measureTeaStoreSecondVariant() { ParserOptions.RESOLVE_BINDINGS_OF_INFERABLE_TYPES.setValue(Boolean.FALSE); ParserOptions.RESOLVE_EVERYTHING.setValue(Boolean.FALSE); ParserOptions.RESOLVE_ALL_BINDINGS.setValue(Boolean.FALSE); - measurePerformance("teastore-second-variant", 20, false); + measurePerformance("teastore-second-variant", 100, false); } @Test @@ -100,8 +129,21 @@ public void printAllAverageTimes() { Files.walk(parentOutput).forEach(path -> { var data = PerformanceData.load(path); System.out.println(path.getFileName().toString()); - System.out.println("Average parsing time: " + data.getAverageParseTime()); - System.out.println("Average resolution time: " + data.getAverageResolutionTime()); + System.out.println("Average parsing time (ms): " + data.getAverageParseTime()); + System.out.println("Average resolution time (ms): " + data.getAverageResolutionTime()); + for (var storage : data.getStorage()) { + System.out.println("Storage (" + + storage.getId() + + "): " + + storage.getCodeFiles() + + " code files of overall " + + storage.getOverallFiles() + + " files taking " + + storage.getTakenStorageByCodeFiles() + + " Bytes for code files of overall " + + storage.getTakenStorage() + + " Bytes."); + } }); } catch (IOException e) { } @@ -123,10 +165,7 @@ private void measurePerformance(String name, int max, boolean fullResolution) { Path target = Paths.get(testInput); JaMoPPJDTSingleFileParser parser = new JaMoPPJDTSingleFileParser(); parser.setExclusionPatterns(".*?src/test/.*?"); - try { - Files.createDirectories(parentOutput); - } catch (IOException e1) { - } + Path outputMeasurement = parentOutput.resolve(name + ".json"); PerformanceData result; if (Files.exists(outputMeasurement)) { @@ -157,28 +196,83 @@ private void measurePerformance(String name, int max, boolean fullResolution) { Set parsedFiles = new HashSet<>(set.getResources()); LOGGER.debug("Asserting the resolution of all proxy objects."); for (Resource res : parsedFiles) { - if (res.getContents().size() == 0 || !(fullResolution && res.getURI().isFile())) { + if (res.getContents().size() == 0 || (!fullResolution && !res.getURI().isFile())) { continue; } this.assertResolveAllProxies(res); } + LOGGER.debug("Reprinting."); for (Resource res : parsedFiles) { - if (res.getContents().size() == 0 || !(fullResolution && res.getURI().isFile())) { + if (res.getContents().size() == 0 || !res.getURI().isFile()) { continue; } + String oldUri = res.getURI().toString(); try { this.testReprint((JavaResource2) res); } catch (Exception e) { - fail(e.getMessage()); + fail(e); } + res.setURI(URI.createURI(oldUri)); } - result.getPoints().add(point); + + result.addPoint(point); PerformanceData.save(result, outputMeasurement); + + if (i == 0 && fullResolution) { + try { + result.setStorage(measureStorage(set)); + } catch (IOException e) { + fail(e); + } + PerformanceData.save(result, outputMeasurement); + } + for (Resource res : parsedFiles) { res.unload(); } } LOGGER.debug("Finished meausring " + name); } + + private List measureStorage(ResourceSet resourceSet) throws IOException { + StoragePerformance javaStorage = new StoragePerformance(); + javaStorage.setId("java"); + var result = OutputUtility.transferToOutput(resourceSet, javaOutput.toString(), "java", true); + fillStorageInformationFromTransfer(javaStorage, javaOutput, result); + + result.sourceTargetMapping().forEach((key, value) -> { + key.getContents().addAll(value.getContents()); + }); + + StoragePerformance xmiStorage = new StoragePerformance(); + xmiStorage.setId("xmi"); + result = OutputUtility.transferToOutput(resourceSet, xmiOutput.toString(), "xmi", true); + fillStorageInformationFromTransfer(xmiStorage, xmiOutput, result); + + result.sourceTargetMapping().forEach((key, value) -> { + key.getContents().addAll(value.getContents()); + }); + + StoragePerformance jsonStorage = new StoragePerformance(); + jsonStorage.setId("json"); + fillStorageInformationFromTransfer(jsonStorage, jsonOutput, OutputUtility.transferToOutput(resourceSet, jsonOutput.toString(), "json", true)); + + return List.of(javaStorage, xmiStorage, jsonStorage); + } + + private void fillStorageInformationFromTransfer(StoragePerformance storage, Path outputDir, TransferResult outputResult) throws IOException { + storage.setTakenStorage(PathUtils.sizeOfDirectory(outputDir)); + long codeFiles = 0; + long codeSize = 0; + for (var entry : outputResult.sourceTargetMapping().entrySet()) { + if (entry.getKey().getURI().isFile()) { + codeFiles++; + codeSize += FileUtils.sizeOf(new File(entry.getValue().getURI().toFileString())); + } + } + storage.setCodeFiles(codeFiles); + storage.setTakenStorageByCodeFiles(codeSize); + storage.setOverallFiles(outputResult.sourceTargetMapping().entrySet().size()); + } } From e1b27f77125f172a5b721b61c295fae94e4f758c Mon Sep 17 00:00:00 2001 From: Martin Armbruster Date: Sun, 30 Jul 2023 18:19:49 +0200 Subject: [PATCH 05/10] The JDT parser only resolves bindings if the JaMoPP parser option is enabled (#23). --- .../jamopp/parser/jdt/singlefile/JaMoPPJDTSingleFileParser.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jamopp.parser.jdt.singlefile/src/main/java/tools/mdsd/jamopp/parser/jdt/singlefile/JaMoPPJDTSingleFileParser.java b/jamopp.parser.jdt.singlefile/src/main/java/tools/mdsd/jamopp/parser/jdt/singlefile/JaMoPPJDTSingleFileParser.java index c9f890d1..97d1564d 100644 --- a/jamopp.parser.jdt.singlefile/src/main/java/tools/mdsd/jamopp/parser/jdt/singlefile/JaMoPPJDTSingleFileParser.java +++ b/jamopp.parser.jdt.singlefile/src/main/java/tools/mdsd/jamopp/parser/jdt/singlefile/JaMoPPJDTSingleFileParser.java @@ -207,7 +207,7 @@ private void setUpResourceSet() { private ASTParser setUpParser() { ASTParser parser = ASTParser.newParser(AST.JLS15); - parser.setResolveBindings(true); + parser.setResolveBindings(ParserOptions.RESOLVE_BINDINGS.isTrue()); parser.setStatementsRecovery(true); Map compilerOptions = new HashMap<>(); compilerOptions.put(JavaCore.COMPILER_SOURCE, JavaCore.VERSION_15); From 5242d4861b4b8bdc6510cb17b8ef92f9204db26c Mon Sep 17 00:00:00 2001 From: Martin Armbruster Date: Sun, 30 Jul 2023 18:20:48 +0200 Subject: [PATCH 06/10] The trivial recovery creates an artificial Object class if it cannot find the model for the Object class (#23). --- .../mdsd/jamopp/recovery/trivial/TrivialRecovery.java | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/jamopp.resolution/src/main/java/tools/mdsd/jamopp/recovery/trivial/TrivialRecovery.java b/jamopp.resolution/src/main/java/tools/mdsd/jamopp/recovery/trivial/TrivialRecovery.java index 1ac6459e..2e25ae72 100644 --- a/jamopp.resolution/src/main/java/tools/mdsd/jamopp/recovery/trivial/TrivialRecovery.java +++ b/jamopp.resolution/src/main/java/tools/mdsd/jamopp/recovery/trivial/TrivialRecovery.java @@ -174,11 +174,18 @@ private void initArtificialResource() { } private tools.mdsd.jamopp.model.java.classifiers.Class findObjectClass() { - return this.set.getResources().stream().filter(resource -> !resource.getContents().isEmpty() + var optionalResult = this.set.getResources().stream().filter(resource -> !resource.getContents().isEmpty() && resource.getContents().get(0) instanceof CompilationUnit) .map(resource -> (CompilationUnit) resource.getContents().get(0)) .filter(cu -> cu.getNamespaces().size() == 2 && cu.getNamespaces().get(0).equals("java") && cu.getNamespaces().get(1).equals("lang") && cu.getName().equals("Object")) - .map(cu -> (tools.mdsd.jamopp.model.java.classifiers.Class) cu.getClassifiers().get(0)).findFirst().get(); + .map(cu -> (tools.mdsd.jamopp.model.java.classifiers.Class) cu.getClassifiers().get(0)).findFirst(); + if (optionalResult.isPresent()) { + return optionalResult.get(); + } + tools.mdsd.jamopp.model.java.classifiers.Class ownObjectClass = ClassifiersFactory.eINSTANCE.createClass(); + ownObjectClass.setName("Object"); + this.artificialCU.getClassifiers().add(ownObjectClass); + return ownObjectClass; } } From 4ef98d518b3d74ad95db3b386995ea7d89e2b232 Mon Sep 17 00:00:00 2001 From: Martin Armbruster Date: Mon, 31 Jul 2023 15:27:16 +0200 Subject: [PATCH 07/10] The class file parser creates blocks for methods and constructors (#23). --- .../tools/mdsd/jamopp/parser/bcel/ClassFileModelLoader.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/jamopp.parser.bcel/src/main/java/tools/mdsd/jamopp/parser/bcel/ClassFileModelLoader.java b/jamopp.parser.bcel/src/main/java/tools/mdsd/jamopp/parser/bcel/ClassFileModelLoader.java index bd450dcd..0576fc43 100644 --- a/jamopp.parser.bcel/src/main/java/tools/mdsd/jamopp/parser/bcel/ClassFileModelLoader.java +++ b/jamopp.parser.bcel/src/main/java/tools/mdsd/jamopp/parser/bcel/ClassFileModelLoader.java @@ -57,6 +57,7 @@ import tools.mdsd.jamopp.model.java.modifiers.ModifiersFactory; import tools.mdsd.jamopp.model.java.parameters.Parameter; import tools.mdsd.jamopp.model.java.parameters.ParametersFactory; +import tools.mdsd.jamopp.model.java.statements.StatementsFactory; import tools.mdsd.jamopp.model.java.types.ClassifierReference; import tools.mdsd.jamopp.model.java.types.TypeReference; import tools.mdsd.jamopp.model.java.types.TypedElement; @@ -215,6 +216,10 @@ private Member constructMethod(org.apache.bcel.classfile.Method method, emfMethod = membersFactory.createClassMethod(); } emfMethod.setName(method.getName()); + + var block = StatementsFactory.eINSTANCE.createBlock(); + block.setName(""); + emfMethod.setStatement(block); String signature = method.getReturnType().getSignature(); String plainSignature = ""; @@ -302,6 +307,7 @@ private Member constructMethod(org.apache.bcel.classfile.Method method, constructor.getTypeParameters().addAll(emfMethod.getTypeParameters()); constructor.getParameters().addAll(emfMethod.getParameters()); constructor.setName(emfClassifier.getName()); + constructor.setBlock(block); return constructor; } From e269a38fcd7064a1bcfcc9bcd0ac43affa5d75e6 Mon Sep 17 00:00:00 2001 From: Martin Armbruster Date: Mon, 31 Jul 2023 15:27:36 +0200 Subject: [PATCH 08/10] The trivial recovery also recovers EnumConstants (#23). --- .../recovery/trivial/TrivialRecovery.java | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/jamopp.resolution/src/main/java/tools/mdsd/jamopp/recovery/trivial/TrivialRecovery.java b/jamopp.resolution/src/main/java/tools/mdsd/jamopp/recovery/trivial/TrivialRecovery.java index 2e25ae72..fdd1d0f7 100644 --- a/jamopp.resolution/src/main/java/tools/mdsd/jamopp/recovery/trivial/TrivialRecovery.java +++ b/jamopp.resolution/src/main/java/tools/mdsd/jamopp/recovery/trivial/TrivialRecovery.java @@ -24,9 +24,11 @@ import tools.mdsd.jamopp.model.java.classifiers.Annotation; import tools.mdsd.jamopp.model.java.classifiers.ClassifiersFactory; +import tools.mdsd.jamopp.model.java.classifiers.Enumeration; import tools.mdsd.jamopp.model.java.containers.CompilationUnit; import tools.mdsd.jamopp.model.java.containers.ContainersFactory; import tools.mdsd.jamopp.model.java.members.ClassMethod; +import tools.mdsd.jamopp.model.java.members.EnumConstant; import tools.mdsd.jamopp.model.java.members.Field; import tools.mdsd.jamopp.model.java.members.InterfaceMethod; import tools.mdsd.jamopp.model.java.members.MembersFactory; @@ -42,6 +44,7 @@ public class TrivialRecovery { private ResourceSet set; private Resource artificialResource; private CompilationUnit artificialCU; + private Enumeration artificialEnum; private tools.mdsd.jamopp.model.java.classifiers.Class artificialClass; private tools.mdsd.jamopp.model.java.classifiers.Class objectClass; private HashMap artClasses = new HashMap<>(); @@ -49,6 +52,7 @@ public class TrivialRecovery { private HashMap artFields = new HashMap<>(); private HashMap artClassMethods = new HashMap<>(); private HashMap artInterfaceMethods = new HashMap<>(); + private HashMap artEnumConstants = new HashMap<>(); private HashMap artPackages = new HashMap<>(); private HashMap artModules = new HashMap<>(); @@ -66,6 +70,9 @@ public void recover() { EList list = (EList) setting.getEObject() .eGet(setting.getEStructuralFeature()); var idx = list.indexOf(proxy); + if (idx == -1) { + continue; + } list.set(idx, actualElement); } else { setting.getEObject().eSet(setting.getEStructuralFeature(), actualElement); @@ -152,6 +159,15 @@ private EObject recoverActualElement(EObject obj) { this.artificialResource.getContents().add(result); this.artModules.put(name, result); return result; + } else if (obj instanceof EnumConstant) { + if (this.artEnumConstants.containsKey(name)) { + return this.artEnumConstants.get(name); + } + var result = MembersFactory.eINSTANCE.createEnumConstant(); + result.setName(name); + this.artificialEnum.getConstants().add(result); + this.artEnumConstants.put(name, result); + return result; } return null; } @@ -168,6 +184,10 @@ private void initArtificialResource() { this.artificialClass.setName("SyntheticClass"); this.artificialCU.getClassifiers().add(this.artificialClass); + this.artificialEnum = ClassifiersFactory.eINSTANCE.createEnumeration(); + this.artificialEnum.setName("SyntheticEnum"); + this.artificialCU.getClassifiers().add(this.artificialEnum); + this.objectClass = findObjectClass(); this.artClasses.put("Object", objectClass); } From 1e06e871f90085c09f8fa82b0c83d64c0cf4b7a0 Mon Sep 17 00:00:00 2001 From: Martin Armbruster Date: Wed, 2 Aug 2023 10:12:05 +0200 Subject: [PATCH 09/10] Added the trivial recovery to the performance test (#23). --- .../performance/PerformanceDataPoint.java | 9 ++++ .../test/performance/PerformanceTest.java | 50 +++++++++++++++---- 2 files changed, 50 insertions(+), 9 deletions(-) diff --git a/jamopp.tests/src/test/java/tools/mdsd/jamopp/test/performance/PerformanceDataPoint.java b/jamopp.tests/src/test/java/tools/mdsd/jamopp/test/performance/PerformanceDataPoint.java index be056cef..02c85f40 100644 --- a/jamopp.tests/src/test/java/tools/mdsd/jamopp/test/performance/PerformanceDataPoint.java +++ b/jamopp.tests/src/test/java/tools/mdsd/jamopp/test/performance/PerformanceDataPoint.java @@ -19,6 +19,7 @@ public class PerformanceDataPoint { private long parseTime; private long resolutionTime; + private long recoverTime; public long getParseTime() { return parseTime; @@ -35,4 +36,12 @@ public long getResolutionTime() { public void setResolutionTime(long resolutionTime) { this.resolutionTime = resolutionTime; } + + public long getRecoverTime() { + return recoverTime; + } + + public void setRecoverTime(long recoverTime) { + this.recoverTime = recoverTime; + } } diff --git a/jamopp.tests/src/test/java/tools/mdsd/jamopp/test/performance/PerformanceTest.java b/jamopp.tests/src/test/java/tools/mdsd/jamopp/test/performance/PerformanceTest.java index aa91f6c9..6756571f 100644 --- a/jamopp.tests/src/test/java/tools/mdsd/jamopp/test/performance/PerformanceTest.java +++ b/jamopp.tests/src/test/java/tools/mdsd/jamopp/test/performance/PerformanceTest.java @@ -39,6 +39,7 @@ import tools.mdsd.jamopp.options.ParserOptions; import tools.mdsd.jamopp.parser.jdt.singlefile.JaMoPPJDTSingleFileParser; +import tools.mdsd.jamopp.recovery.trivial.TrivialRecovery; import tools.mdsd.jamopp.resource.JavaResource2; import tools.mdsd.jamopp.test.AbstractJaMoPPTests; import tools.mdsd.jamopp.test.OutputUtility; @@ -84,7 +85,7 @@ public void measureTeaStoreFullResolution() { ParserOptions.RESOLVE_BINDINGS_OF_INFERABLE_TYPES.setValue(Boolean.TRUE); ParserOptions.RESOLVE_EVERYTHING.setValue(Boolean.TRUE); ParserOptions.RESOLVE_ALL_BINDINGS.setValue(Boolean.TRUE); - measurePerformance("teastore-full-resolution", 100, true); + measurePerformance("teastore-full-resolution", 100, true, false); } @Test @@ -96,11 +97,10 @@ public void measureTeaStoreWithoutResolvingEverything() { ParserOptions.RESOLVE_BINDINGS_OF_INFERABLE_TYPES.setValue(Boolean.TRUE); ParserOptions.RESOLVE_EVERYTHING.setValue(Boolean.FALSE); ParserOptions.RESOLVE_ALL_BINDINGS.setValue(Boolean.TRUE); - measurePerformance("teastore-without-resolving-everything", 100, true); + measurePerformance("teastore-without-resolving-everything", 100, true, false); } - @Test - public void measureTeaStoreWithOneLevelResolution() { + private void prepareParserOptionsForOneLevelResolution() { ParserOptions.CREATE_LAYOUT_INFORMATION.setValue(Boolean.TRUE); ParserOptions.REGISTER_LOCAL.setValue(Boolean.TRUE); ParserOptions.PREFER_BINDING_CONVERSION.setValue(Boolean.TRUE); @@ -108,11 +108,22 @@ public void measureTeaStoreWithOneLevelResolution() { ParserOptions.RESOLVE_BINDINGS_OF_INFERABLE_TYPES.setValue(Boolean.TRUE); ParserOptions.RESOLVE_EVERYTHING.setValue(Boolean.FALSE); ParserOptions.RESOLVE_ALL_BINDINGS.setValue(Boolean.FALSE); - measurePerformance("teastore-one-level-resolution", 100, false); } @Test - public void measureTeaStoreSecondVariant() { + public void measureTeaStoreWithOneLevelResolution() { + prepareParserOptionsForOneLevelResolution(); + measurePerformance("teastore-one-level-resolution", 100, false, true); + } + + @Disabled("Takes several hours.") + @Test + public void measureTeaStoreWithOneLevelResolutionAndFullResolution() { + prepareParserOptionsForOneLevelResolution(); + measurePerformance("teastore-one-level-resolution-full", 1, true, false); + } + + private void prepareParserOptionsForSecondVariant() { ParserOptions.CREATE_LAYOUT_INFORMATION.setValue(Boolean.TRUE); ParserOptions.REGISTER_LOCAL.setValue(Boolean.TRUE); ParserOptions.PREFER_BINDING_CONVERSION.setValue(Boolean.TRUE); @@ -120,7 +131,19 @@ public void measureTeaStoreSecondVariant() { ParserOptions.RESOLVE_BINDINGS_OF_INFERABLE_TYPES.setValue(Boolean.FALSE); ParserOptions.RESOLVE_EVERYTHING.setValue(Boolean.FALSE); ParserOptions.RESOLVE_ALL_BINDINGS.setValue(Boolean.FALSE); - measurePerformance("teastore-second-variant", 100, false); + } + + @Test + public void measureTeaStoreSecondVariant() { + prepareParserOptionsForSecondVariant(); + measurePerformance("teastore-second-variant", 1, false, true); + } + + @Disabled("Takes several hours.") + @Test + public void measureTeaStoreSecondVariantAndFullResolution() { + prepareParserOptionsForSecondVariant(); + measurePerformance("teastore-second-variant-resolution", 1, true, false); } @Test @@ -159,7 +182,7 @@ protected String getTestInputFolder() { return inputFolder; } - private void measurePerformance(String name, int max, boolean fullResolution) { + private void measurePerformance(String name, int max, boolean fullResolution, boolean recover) { String testInput = getTestInputFolder(); LOGGER.debug("Executing performance measurements for " + name); Path target = Paths.get(testInput); @@ -175,6 +198,7 @@ private void measurePerformance(String name, int max, boolean fullResolution) { } int actualMax = Math.min(max, max - result.getPoints().size()); for (int i = 0; i < actualMax; i++) { + System.out.println("Measurement " + i + " for " + name); PerformanceDataPoint point = new PerformanceDataPoint(); long millis = System.currentTimeMillis(); ResourceSet set = parser.parseDirectory(target); @@ -193,6 +217,14 @@ private void measurePerformance(String name, int max, boolean fullResolution) { millis = System.currentTimeMillis() - millis; } point.setResolutionTime(millis); + + if (recover) { + millis = System.currentTimeMillis(); + new TrivialRecovery(set).recover(); + millis = System.currentTimeMillis() - millis; + point.setRecoverTime(millis); + } + Set parsedFiles = new HashSet<>(set.getResources()); LOGGER.debug("Asserting the resolution of all proxy objects."); for (Resource res : parsedFiles) { @@ -219,7 +251,7 @@ private void measurePerformance(String name, int max, boolean fullResolution) { result.addPoint(point); PerformanceData.save(result, outputMeasurement); - if (i == 0 && fullResolution) { + if (i == 0 && (fullResolution || recover)) { try { result.setStorage(measureStorage(set)); } catch (IOException e) { From f077fbfd1206cd3245901e8bd44bc62ea15bda45 Mon Sep 17 00:00:00 2001 From: Martin Armbruster Date: Wed, 2 Aug 2023 10:38:55 +0200 Subject: [PATCH 10/10] Updated the CHANEGLOG (#23). --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index fe93a77a..dee84b23 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), - Trivial recovery strategy to generate model elements for unresolved proxy objects - Parser: `TextBlock`s are converted to `TextBockReference`s so that model elements are generated for text blocks +- Performance Test: + - Performs trivial recovery + - Measures model storage ### Changed @@ -34,6 +37,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), ### Fixed - First variant: always returns an empty model (temporary fix to not end in an endless loop) +- Class file parser: creates bodies for methods and constructors ### Security