Pool filesystems during testing to reduce the amount of memory used d…

…uring testing.

common/src/main/java/org/fao/geonet/utils/IO.java

@@ -39,12 +39,17 @@
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.nio.file.SimpleFileVisitor;
+import java.nio.file.StandardCopyOption;
 import java.nio.file.attribute.BasicFileAttributes;
 import java.nio.file.attribute.FileTime;
 import java.sql.ResultSet;
 import java.sql.Statement;
+import java.util.Comparator;
+import java.util.HashMap;
 import java.util.Iterator;
+import java.util.Map;
 import java.util.Objects;
+import java.util.TreeSet;
 import java.util.concurrent.TimeUnit;
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
@@ -258,44 +263,69 @@ public static void setFileSystemThreadLocal(FileSystem newFileSystem) {
     }
 
     public static void printDirectoryTree(Path dir, final boolean printFileSize) throws IOException {
-        Files.walkFileTree(dir, new SimpleFileVisitor<Path>(){
+        final Map<Path, Long> fileSizes = new HashMap<>();
+        final TreeSet<Path> largestFiles = new TreeSet<>(new Comparator<Path>() {
+            @Override
+            public int compare(Path o1, Path o2) {
+                return Long.compare(fileSizes.get(o2), fileSizes.get(o1));
+            }
+        });
+
+        Files.walkFileTree(dir, new SimpleFileVisitor<Path>() {
             int depth = 0;
+            long dirSize = 0;
 
             @Override
             public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
                 StringBuilder builder = newStringBuilder();
                 builder.append("> ").append(dir.getFileName()).append(" (").append(dir.getParent()).append("):");
                 System.out.println(builder.toString());
-                depth += 4;
+                depth += 1;
+                dirSize = 0;
                 return FileVisitResult.CONTINUE;
             }
 
             @Override
             public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
                 StringBuilder builder = newStringBuilder();
                 builder.append("- ").append(file.getFileName());
+                final long fileSize = Files.size(file);
+                dirSize += fileSize;
+                fileSizes.put(file, fileSize);
                 if (printFileSize) {
                     final int KB = 1024;
-                    builder.append(" (").append(Files.size(file) / KB).append(" kb)");
+                    builder.append(" (").append(fileSize / KB).append(" kb)");
                 }
+                largestFiles.add(file);
                 System.out.println(builder.toString());
                 return FileVisitResult.CONTINUE;
             }
 
             protected StringBuilder newStringBuilder() {
                 StringBuilder builder = new StringBuilder();
-                for (int j = 0; j < depth; j++) {
+                final int indent = depth * 4;
+                for (int j = 0; j < indent; j++) {
                     builder.append(" ");
                 }
                 return builder;
             }
 
             @Override
             public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
-                depth -=4;
+                fileSizes.put(dir, dirSize);
+                depth -= 1;
                 return FileVisitResult.CONTINUE;
             }
         });
+
+        final StringBuilder summary = new StringBuilder("The following files/directories are the 10 largest.");
+        summary.append("A directory's size is calculated sum of the files (not directories) it contains.");
+        Iterator<Path> iter = largestFiles.iterator();
+        for (int i = 0; i < 10 && iter.hasNext(); i++) {
+            final Path path = iter.next();
+            summary.append("\n\t - ").append(path).append(": ").append(fileSizes.get(path));
+        }
+        System.out.println(summary);
     }
 
     public static URL toURL(Path textFile) throws MalformedURLException {
@@ -347,7 +377,7 @@ public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) th
 
         @Override
         public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
-            Files.copy(file, relativeFile(from, file, actualTo));
+            Files.copy(file, relativeFile(from, file, actualTo), StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.COPY_ATTRIBUTES);
             return FileVisitResult.CONTINUE;
         }
     }
@@ -364,14 +394,16 @@ public CopyAcceptedFiles(@Nonnull Path from, @Nonnull Path actualTo, @Nonnull Di
 
         @Override
         public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
-            Files.createDirectories(relativeFile(from, dir, actualTo));
+            final Path newDir = relativeFile(from, dir, actualTo);
+            Files.createDirectories(newDir);
+            Files.setLastModifiedTime(newDir, Files.getLastModifiedTime(dir));
             return FileVisitResult.CONTINUE;
         }
 
         @Override
         public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
             if (filter.accept(file)) {
-                Files.copy(file, relativeFile(from, file, actualTo));
+                Files.copy(file, relativeFile(from, file, actualTo), StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.COPY_ATTRIBUTES);
             }
             return FileVisitResult.CONTINUE;
         }

core/src/main/java/org/fao/geonet/kernel/GeonetworkDataDirectory.java

@@ -327,7 +327,15 @@ private void initDataDirectory() throws IOException {
                 final Path srcLogo = this.webappDir.resolve("images").resolve("harvesting");
 
                 if (Files.exists(srcLogo)) {
-                    IO.copyDirectoryOrFile(srcLogo, logoDir, false);
+                    try (DirectoryStream<Path> paths = Files.newDirectoryStream(srcLogo)) {
+                        for (Path path : paths) {
+                            final Path relativePath = srcLogo.relativize(path);
+                            final Path dest = logoDir.resolve(relativePath.toString());
+                            if (!Files.exists(dest)) {
+                                IO.copyDirectoryOrFile(path, dest, false);
+                            }
+                        }
+                    }
                 }
 
             } catch (IOException e) {

core/src/test/java/org/fao/geonet/FileSystemPool.java

@@ -0,0 +1,137 @@
+package org.fao.geonet;
+
+import com.google.common.jimfs.Configuration;
+import com.google.common.jimfs.Jimfs;
+import org.fao.geonet.utils.IO;
+
+import java.io.IOException;
+import java.nio.file.FileSystem;
+import java.nio.file.FileVisitResult;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.StandardCopyOption;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.util.Stack;
+
+import static java.nio.file.Files.exists;
+import static java.nio.file.Files.getLastModifiedTime;
+
+/**
+ * Responsible for obtaining in memory filesystems for tests.  In order to reduce memory consumption this class will recycle
+ * filesystems.  The reason this is needed is because when each test obtains a new FS and copies all the files to the new FS, then a
+ * great deal of memory is consumed and then needs to be garbage collected.  This can result in 2 types of OOM errors.  One is a heap
+ * space error where all the heap space is used up.  The other is a GC overhead exceeded error where there is too much garbage collection
+ * and the JVM throws an error regarding this.
+ * <p/>
+ * This class also synchronizes each filesystem that is obtained with the template file system by ensuring that all files are the same
+ * as the files in the template.
+ *
+ * @author Jesse on 11/24/2014.
+ */
+public class FileSystemPool {
+    private static final int MAX_FS = 5;
+    private final CreatedFs template = new CreatedFs(Jimfs.newFileSystem("template", Configuration.unix()), "/", "data");
+    private int openFs = 0;
+    private Stack<CreatedFs> pool = new Stack<>();
+
+
+    public synchronized CreatedFs getTemplate() {
+        return template;
+    }
+    public synchronized CreatedFs get(String fsId) throws IOException {
+        while (openFs > MAX_FS && pool.isEmpty()) {
+            try {
+                this.wait(1000);
+            } catch (InterruptedException e) {
+                Jimfs.newFileSystem(fsId, Configuration.unix());
+            }
+        }
+
+        final CreatedFs fileSystem;
+        if (pool.isEmpty()) {
+            openFs++;
+            fileSystem = new CreatedFs(Jimfs.newFileSystem(fsId, Configuration.unix()), "nodes", "default_data_dir");
+        } else {
+            fileSystem = pool.pop();
+        }
+
+        syncWithTemplate(fileSystem);
+        IO.setFileSystemThreadLocal(fileSystem.fs);
+
+        return fileSystem;
+    }
+
+    public synchronized void release(CreatedFs fs) {
+        pool.add(fs);
+    }
+
+
+    private void syncWithTemplate(CreatedFs fileSystem) throws IOException {
+        removeModifiedFiles(fileSystem);
+        copyMissingFiles(fileSystem);
+    }
+
+    private void removeModifiedFiles(final CreatedFs fileSystem) throws IOException {
+        if (!exists(fileSystem.dataDir)) {
+            return;
+        }
+
+        Files.walkFileTree(fileSystem.dataDir, new SimpleFileVisitor<Path>(){
+            @Override
+            public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
+                Path relativePath = fileSystem.dataDir.relativize(dir);
+
+                if (!exists(template.dataDir.resolve(relativePath.toString()))) {
+                    IO.deleteFileOrDirectory(dir);
+                    return FileVisitResult.SKIP_SUBTREE;
+                }
+
+                return FileVisitResult.CONTINUE;
+            }
+
+            @Override
+            public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
+                Path templatePath = template.dataDir.resolve(fileSystem.dataDir.relativize(file).toString());
+                if (!exists(templatePath)) {
+                    Files.delete(file);
+                }
+                return super.visitFile(file, attrs);
+            }
+        });
+    }
+
+    private void copyMissingFiles(final CreatedFs fileSystem) throws IOException {
+        Files.walkFileTree(template.dataDir, new SimpleFileVisitor<Path>(){
+            @Override
+            public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
+                final Path newDir = fileSystem.dataDir.resolve(template.dataDir.relativize(dir).toString());
+                Files.createDirectories(newDir);
+                Files.setLastModifiedTime(newDir, Files.getLastModifiedTime(dir));
+
+                return super.preVisitDirectory(dir, attrs);
+            }
+
+            @Override
+            public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
+                Path newFsPath = fileSystem.dataDir.resolve(template.dataDir.relativize(file).toString());
+                if (!exists(newFsPath) || getLastModifiedTime(file).toMillis() < getLastModifiedTime(newFsPath).toMillis()) {
+                    Files.copy(file, newFsPath, StandardCopyOption.COPY_ATTRIBUTES, StandardCopyOption.REPLACE_EXISTING);
+                }
+                return super.visitFile(file, attrs);
+            }
+        });
+    }
+
+    public class CreatedFs {
+        public final FileSystem fs;
+        public final Path dataDirContainer;
+        public final Path dataDir;
+
+        public CreatedFs(FileSystem fs, String dataDirContainer, String dataDir) {
+            this.fs = fs;
+            this.dataDirContainer = fs.getPath(dataDirContainer);
+            this.dataDir = this.dataDirContainer.resolve(dataDir);
+        }
+    }
+}