From fdb7ac352c5fb7eb69a436a5b4dc42c8d9d0a268 Mon Sep 17 00:00:00 2001 From: rodrigopardomeza Date: Tue, 31 Aug 2021 00:00:51 +0200 Subject: [PATCH] [WAYANG-32] More general corrections, comments, and Readme update --- wayang-commons/pom.xml | 3 +- .../wayang-utils/wayang-profile-db/readme.md | 31 +++- .../src/main/java/profiledb/ProfileDB.java | 44 ++++++ .../main/java/profiledb/storage/Storage.java | 46 +++++- .../test/java/profiledb/ProfileDBTest.java | 135 ++++++++---------- 5 files changed, 172 insertions(+), 87 deletions(-) diff --git a/wayang-commons/pom.xml b/wayang-commons/pom.xml index 4dab9d5bd..2caa9603a 100644 --- a/wayang-commons/pom.xml +++ b/wayang-commons/pom.xml @@ -214,7 +214,7 @@ com.google.code.gson gson - 2.2.4 + 2.8.8 @@ -222,6 +222,7 @@ wayang-core wayang-basic + wayang-utils \ No newline at end of file diff --git a/wayang-commons/wayang-utils/wayang-profile-db/readme.md b/wayang-commons/wayang-utils/wayang-profile-db/readme.md index 8c14afcf8..0e1fe47c7 100644 --- a/wayang-commons/wayang-utils/wayang-profile-db/readme.md +++ b/wayang-commons/wayang-utils/wayang-profile-db/readme.md @@ -1,3 +1,30 @@ -Base on +This code is based on the implementation you can find in the following repository: -https://github.com/sekruse/profiledb-java.git \ No newline at end of file +- https://github.com/sekruse/profiledb-java.git + +The code there does not have regular maintenance. Wayang will require new functionalities to deal with serialization of UDFs and storage in other platforms. + +The classes below has not been modified: + + MeasurementDeserializer + MeasurementSerializer + Experiment + Measurement + Subject + TimeMeasurement + Type + +The classes below has been added/modified to provide an abstraction over different Storage methods: + + ProfileDB + Storage + FileStorage + JDBCStorage + +The code that is based on the mentioned repository starts and ends with the commits indicated below: + +- start: [5344336f68bb9038e701435e9859321b6e8cbcfc](https://github.com/apache/incubator-wayang/commit/5344336f68bb9038e701435e9859321b6e8cbcfc) + +- end: + +All the code that has been added after those commits is totally independent of the mentioned repository. \ No newline at end of file diff --git a/wayang-commons/wayang-utils/wayang-profile-db/src/main/java/profiledb/ProfileDB.java b/wayang-commons/wayang-utils/wayang-profile-db/src/main/java/profiledb/ProfileDB.java index 21490e95a..302023377 100644 --- a/wayang-commons/wayang-utils/wayang-profile-db/src/main/java/profiledb/ProfileDB.java +++ b/wayang-commons/wayang-utils/wayang-profile-db/src/main/java/profiledb/ProfileDB.java @@ -40,30 +40,74 @@ public class ProfileDB { */ private Gson gson; + /** + * Receive an array of {@link Experiment}s and persist them + * + * @param experiments Array of {@link Experiment}s to be persisted + * @throws IOException + */ public void save(Experiment... experiments) throws IOException { this.storage.save(experiments); } + /** + * Receive a Collection of {@link Experiment}s and persist them + * + * @param experiments Collection of {@link Experiment}s to be persisted + * @throws IOException + */ public void save(Collection experiments) throws IOException { this.storage.save(experiments); } + /** + * Receive a Collection of {@link Experiment}s and persist them + * + * @param experiments Collection of {@link Experiment}s to be persisted + * @param outputStream Indicates where the data must to be written + * @throws IOException + */ public void save(Collection experiments, OutputStream outputStream) throws IOException { this.storage.save(experiments, outputStream); } + /** + * Related to file based storage, Receive an array of {@link Experiment}s and persist them at the end of a file + * + * @param experiments Array of {@link Experiment}s to be persisted + * @throws IOException + */ public void append(Experiment... experiments) throws IOException { this.storage.append(experiments); } + /** + * Related to file based storage, Receive a Collection of {@link Experiment}s and persist them at the end of a file + * + * @param experiments Collection of {@link Experiment}s to be persisted + * @throws IOException + */ public void append(Collection experiments) throws IOException { this.storage.append(experiments); } + /** + * Bring {@link Experiment}s from current Storage to local variable + * + * @return Collection of {@link Experiment}s + * @throws IOException + */ public Collection load() throws IOException { return this.storage.load(); } + /** + * Bring {@link Experiment}s from current Storage to local variable + * + * @param inputStream Data to be read + * @return Collection of {@link Experiment}s + * @throws IOException + */ public Collection load(InputStream inputStream) throws IOException { return this.storage.load(inputStream); } diff --git a/wayang-commons/wayang-utils/wayang-profile-db/src/main/java/profiledb/storage/Storage.java b/wayang-commons/wayang-utils/wayang-profile-db/src/main/java/profiledb/storage/Storage.java index b27aefc66..f89f9c619 100644 --- a/wayang-commons/wayang-utils/wayang-profile-db/src/main/java/profiledb/storage/Storage.java +++ b/wayang-commons/wayang-utils/wayang-profile-db/src/main/java/profiledb/storage/Storage.java @@ -32,6 +32,8 @@ public Storage(URI uri){ this.storageFile = uri; } + + /** * Sets the ProfileDB for this instance that manages all the Measurement subclasses * */ @@ -47,18 +49,48 @@ public void changeLocation(URI uri){ this.storageFile = uri; } - public void save(Experiment... experiments) throws IOException { - System.out.println("llegue"); - } + /** + * Receive an array of {@link Experiment}s and persist them + * + * @param experiments Array of {@link Experiment}s to be persisted + * @throws IOException + */ + public abstract void save(Experiment... experiments) throws IOException; - public void save(Collection experiments) throws IOException {} + /** + * Receive a Collection of {@link Experiment}s and persist them + * + * @param experiments Collection of {@link Experiment}s to be persisted + * @throws IOException + */ + public abstract void save(Collection experiments) throws IOException; - public void append(Experiment... experiments) throws IOException {} + /** + * Related to file based storage, Receive an array of {@link Experiment}s and persist them at the end of a file + * + * @param experiments Array of {@link Experiment}s to be persisted + * @throws IOException + */ + public abstract void append(Experiment... experiments) throws IOException; - public void append(Collection experiments) throws IOException {} + /** + * Related to file based storage, Receive a Collection of {@link Experiment}s and persist them at the end of a file + * + * @param experiments Collection of {@link Experiment}s to be persisted + * @throws IOException + */ + public abstract void append(Collection experiments) throws IOException ; + + /** + * Bring {@link Experiment}s from current Storage to local variable + * + * @return Collection of {@link Experiment}s + * @throws IOException + */ + public abstract Collection load() throws IOException; - public Collection load() throws IOException { return null; } + //TODO The following methods should be moved to file storage implementation /** * Write {@link Experiment}s to an {@link OutputStream}. * diff --git a/wayang-commons/wayang-utils/wayang-profile-db/src/test/java/profiledb/ProfileDBTest.java b/wayang-commons/wayang-utils/wayang-profile-db/src/test/java/profiledb/ProfileDBTest.java index ea93bc750..b002c6505 100644 --- a/wayang-commons/wayang-utils/wayang-profile-db/src/test/java/profiledb/ProfileDBTest.java +++ b/wayang-commons/wayang-utils/wayang-profile-db/src/test/java/profiledb/ProfileDBTest.java @@ -15,7 +15,9 @@ import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; +import java.net.URL; import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.Paths; import java.util.*; @@ -25,32 +27,28 @@ public class ProfileDBTest { public void testPolymorphSaveAndLoad() throws IOException { try { - URI uri = new URI("file:///Users/rodrigopardomeza/Desktop/random/myfile.txt"); + Path temp = Files.createTempFile("", ".tmp"); + + String absolutePath = temp.toString(); + System.out.println("Temp file : " + absolutePath); + + URI uri = new URI("my-file4"); + FileStorage store = new FileStorage(uri); ProfileDB profileDB = new ProfileDB(store) .registerMeasurementClass(TestMemoryMeasurement.class) .registerMeasurementClass(TestTimeMeasurement.class); - /** - * Esto es lo que se espera del codigo del cliente - * Tiene que usar la API para registrar medidas - */ - // crea un experimento falso final Experiment experiment = new Experiment("test-xp", new Subject("PageRank", "1.0"), "test experiment"); - // Agrega medidas falsas hardcoded Measurement timeMeasurement = new TestTimeMeasurement("exec-time", 12345L); Measurement memoryMeasurement = new TestMemoryMeasurement("exec-time", System.currentTimeMillis(), 54321L); - /*Agrega las medidas al experimento*/ experiment.addMeasurement(timeMeasurement); experiment.addMeasurement(memoryMeasurement); // Save the experiment. - /** - * Guarda el experimento en memoria - */ byte[] buffer; ByteArrayOutputStream bos = new ByteArrayOutputStream(); profileDB.save(Collections.singleton(experiment), bos); @@ -59,9 +57,6 @@ public void testPolymorphSaveAndLoad() throws IOException { System.out.println("Buffer contents: " + new String(buffer, "UTF-8")); // Load the experiment. - /** - * Lee el experimento desde el buffer en memoria - */ ByteArrayInputStream bis = new ByteArrayInputStream(buffer); Collection loadedExperiments = profileDB.load(bis); @@ -86,7 +81,7 @@ public void testPolymorphSaveAndLoad() throws IOException { @Test public void testRecursiveSaveAndLoad() throws IOException { try { - URI uri = new URI("file:///Users/rodrigopardomeza/Desktop/random/myfile.txt"); + URI uri = new URI("my-file2"); FileStorage store = new FileStorage(uri); ProfileDB profileDB = new ProfileDB(store) @@ -128,78 +123,64 @@ public void testRecursiveSaveAndLoad() throws IOException { @Test public void testFileOperations() throws IOException { - - try { - URI uri = new URI("file:///Users/rodrigopardomeza/Desktop/random/myfile.txt"); - FileStorage store = new FileStorage(uri); - - ProfileDB profileDB = new ProfileDB(store) - .registerMeasurementClass(TestMemoryMeasurement.class) - .registerMeasurementClass(TestTimeMeasurement.class); - - // Create example experiments. - final Experiment experiment1 = new Experiment("xp1", new Subject("PageRank", "1.0"), "test experiment 1"); - experiment1.addMeasurement(new TestTimeMeasurement("exec-time", 1L)); - final Experiment experiment2 = new Experiment("xp2", new Subject("KMeans", "1.1"), "test experiment 2"); - experiment2.addMeasurement(new TestTimeMeasurement("exec-time", 2L)); - final Experiment experiment3 = new Experiment("xp3", new Subject("Apriori", "2.0"), "test experiment 3"); - experiment3.addMeasurement(new TestMemoryMeasurement("ram", System.currentTimeMillis(), 3L)); - - // Save the experiments. - File tempDir = Files.createTempDirectory("profiledb").toFile(); - //File dir = Files.createTempDirectory(Paths.get("/Users/rodrigopardomeza/Desktop/random/"), "profiledb").toFile(); - //File dir = Paths.get("/Users/rodrigopardomeza/Desktop/random/").toFile(); - File file = new File(tempDir, "profiledb.json"); - file.createNewFile(); - profileDB.save(experiment1); - profileDB.append(experiment2, experiment3); - - System.out.println("File plat" + file.toPath().toUri().toString()); - Files.lines(file.toPath()).forEach(System.out::println); - - // Load and compare. - final Set loadedExperiments = new HashSet<>(profileDB.load()); - final List expectedExperiments = Arrays.asList(experiment1, experiment2, experiment3); - Assert.assertEquals(expectedExperiments.size(), loadedExperiments.size()); - Assert.assertEquals(new HashSet<>(expectedExperiments), new HashSet<>(loadedExperiments)); - } catch (URISyntaxException e) { - e.printStackTrace(); - } + File tempDir = Files.createTempDirectory("profiledb").toFile(); + File file = new File(tempDir, "profiledb.json"); + file.createNewFile(); + FileStorage store = new FileStorage(file.toURI()); + + ProfileDB profileDB = new ProfileDB(store) + .registerMeasurementClass(TestMemoryMeasurement.class) + .registerMeasurementClass(TestTimeMeasurement.class); + + // Create example experiments. + final Experiment experiment1 = new Experiment("xp1", new Subject("PageRank", "1.0"), "test experiment 1"); + experiment1.addMeasurement(new TestTimeMeasurement("exec-time", 1L)); + final Experiment experiment2 = new Experiment("xp2", new Subject("KMeans", "1.1"), "test experiment 2"); + experiment2.addMeasurement(new TestTimeMeasurement("exec-time", 2L)); + final Experiment experiment3 = new Experiment("xp3", new Subject("Apriori", "2.0"), "test experiment 3"); + experiment3.addMeasurement(new TestMemoryMeasurement("ram", System.currentTimeMillis(), 3L)); + + // Save the experiments. + profileDB.save(experiment1); + profileDB.append(experiment2, experiment3); + + Files.lines(file.toPath()).forEach(System.out::println); + + // Load and compare. + final Set loadedExperiments = new HashSet<>(profileDB.load()); + final List expectedExperiments = Arrays.asList(experiment1, experiment2, experiment3); + Assert.assertEquals(expectedExperiments.size(), loadedExperiments.size()); + Assert.assertEquals(new HashSet<>(expectedExperiments), new HashSet<>(loadedExperiments)); } @Test public void testAppendOnNonExistentFile() throws IOException { - try { - URI uri = new URI("file:///Users/rodrigopardomeza/Desktop/random/myfile.txt"); - FileStorage store = new FileStorage(uri); + File tempDir = Files.createTempDirectory("profiledb").toFile(); + File file = new File(tempDir, "new-profiledb.json"); + file.createNewFile(); + FileStorage store = new FileStorage(file.toURI()); - // This seems to be an issue on Linux. - ProfileDB profileDB = new ProfileDB(store) - .registerMeasurementClass(TestMemoryMeasurement.class) - .registerMeasurementClass(TestTimeMeasurement.class); + // This seems to be an issue on Linux. + ProfileDB profileDB = new ProfileDB(store) + .registerMeasurementClass(TestMemoryMeasurement.class) + .registerMeasurementClass(TestTimeMeasurement.class); - // Create example experiments. - final Experiment experiment1 = new Experiment("xp1", new Subject("PageRank", "1.0"), "test experiment 1"); - experiment1.addMeasurement(new TestTimeMeasurement("exec-time", 1L)); + // Create example experiments. + final Experiment experiment1 = new Experiment("xp1", new Subject("PageRank", "1.0"), "test experiment 1"); + experiment1.addMeasurement(new TestTimeMeasurement("exec-time", 1L)); - // Save the experiments. - File tempDir = Files.createTempDirectory("profiledb").toFile(); - File file = new File(tempDir, "new-profiledb.json"); - file.createNewFile(); - Assert.assertTrue(!file.exists() || file.delete()); - profileDB.append(experiment1); + // Save the experiments. + Assert.assertTrue(!file.exists() || file.delete()); + profileDB.append(experiment1); - Files.lines(file.toPath()).forEach(System.out::println); + Files.lines(file.toPath()).forEach(System.out::println); - // Load and compare. - final Set loadedExperiments = new HashSet<>(profileDB.load()); - final List expectedExperiments = Collections.singletonList(experiment1); - Assert.assertEquals(expectedExperiments.size(), loadedExperiments.size()); - Assert.assertEquals(new HashSet<>(expectedExperiments), new HashSet<>(loadedExperiments)); - } catch (URISyntaxException e) { - e.printStackTrace(); - } + // Load and compare. + final Set loadedExperiments = new HashSet<>(profileDB.load()); + final List expectedExperiments = Collections.singletonList(experiment1); + Assert.assertEquals(expectedExperiments.size(), loadedExperiments.size()); + Assert.assertEquals(new HashSet<>(expectedExperiments), new HashSet<>(loadedExperiments)); } }