Create dump/load facility

community/dbms/LICENSES.txt

@@ -3,6 +3,7 @@ libraries. For an overview of the licenses see the NOTICE.txt file.
 
 ------------------------------------------------------------------------------
 Apache Software License, Version 2.0
+  Apache Commons Compress
   Apache Commons Lang
   Lucene Core
   Lucene Memory

community/dbms/NOTICE.txt

@@ -26,6 +26,7 @@ Third-party licenses
 --------------------
 
 Apache Software License, Version 2.0
+  Apache Commons Compress
   Apache Commons Lang
   Lucene Core
   Lucene Memory

community/dbms/pom.xml

@@ -54,7 +54,18 @@
       <artifactId>neo4j-kernel</artifactId>
       <version>${project.version}</version>
     </dependency>
+    <dependency>
+      <groupId>org.apache.commons</groupId>
+      <artifactId>commons-compress</artifactId>
+    </dependency>
 
+    <dependency>
+      <groupId>org.neo4j</groupId>
+      <artifactId>neo4j-io</artifactId>
+      <version>${project.version}</version>
+      <type>test-jar</type>
+      <scope>test</scope>
+    </dependency>
     <dependency>
       <groupId>junit</groupId>
       <artifactId>junit</artifactId>

community/dbms/src/main/java/org/neo4j/dbms/archive/Dumper.java

@@ -0,0 +1,111 @@
+/*
+ * Copyright (c) 2002-2016 "Neo Technology,"
+ * Network Engine for Objects in Lund AB [http://neotechnology.com]
+ *
+ * This file is part of Neo4j.
+ *
+ * Neo4j is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.neo4j.dbms.archive;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+import java.util.stream.Stream;
+
+import org.apache.commons.compress.archivers.ArchiveEntry;
+import org.apache.commons.compress.archivers.ArchiveOutputStream;
+import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
+import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream;
+
+import static java.nio.file.Files.isRegularFile;
+
+import static org.neo4j.dbms.archive.Utils.checkWritableDirectory;
+import static org.neo4j.dbms.archive.Utils.copy;
+
+public class Dumper
+{
+    public void dump( Path root, Path archive ) throws IOException
+    {
+        checkWritableDirectory( archive.getParent() );
+        try ( Stream<Path> files = Files.walk( root );
+              ArchiveOutputStream stream = openArchiveOut( archive ) )
+        {
+            files.forEach( file -> dumpFile( file, root, stream ) );
+        }
+        catch ( TunnellingException e )
+        {
+            throw e.getWrapped();
+        }
+    }
+
+    private static ArchiveOutputStream openArchiveOut( Path archive ) throws IOException
+    {
+        // StandardOpenOption.CREATE_NEW is important here because it atomically asserts that the file doesn't
+        // exist as it is opened, avoiding a TOCTOU race condition which results in a security vulnerability. I
+        // can't see a way to write a test to verify that we are using this option rather than just implementing
+        // the check ourselves non-atomically.
+        TarArchiveOutputStream tarball =
+                new TarArchiveOutputStream( new GzipCompressorOutputStream(
+                        Files.newOutputStream( archive, StandardOpenOption.CREATE_NEW ) ) );
+        tarball.setLongFileMode( TarArchiveOutputStream.LONGFILE_POSIX );
+        return tarball;
+    }
+
+    private static void dumpFile( Path file, Path root, ArchiveOutputStream archive )
+    {
+        try
+        {
+            ArchiveEntry entry = createEntry( file, root, archive );
+            archive.putArchiveEntry( entry );
+            if ( isRegularFile( file ) )
+            {
+                writeFile( file, archive );
+            }
+            archive.closeArchiveEntry();
+        }
+        catch ( IOException e )
+        {
+            throw new TunnellingException( e );
+        }
+    }
+
+    private static ArchiveEntry createEntry( Path file, Path root, ArchiveOutputStream archive ) throws IOException
+    {
+        return archive.createArchiveEntry( file.toFile(), "./" + root.relativize( file ).toString() );
+    }
+
+    private static void writeFile( Path file, ArchiveOutputStream archiveStream ) throws IOException
+    {
+        try ( InputStream in = Files.newInputStream( file ) )
+        {
+            copy( in, archiveStream );
+        }
+    }
+
+    private static class TunnellingException extends RuntimeException
+    {
+        public TunnellingException( IOException e )
+        {
+            super( e );
+        }
+
+        public IOException getWrapped()
+        {
+            return (IOException) getCause();
+        }
+    }
+}

community/dbms/src/main/java/org/neo4j/dbms/archive/IncorrectFormat.java

@@ -0,0 +1,31 @@
+/*
+ * Copyright (c) 2002-2016 "Neo Technology,"
+ * Network Engine for Objects in Lund AB [http://neotechnology.com]
+ *
+ * This file is part of Neo4j.
+ *
+ * Neo4j is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.neo4j.dbms.archive;
+
+import java.io.IOException;
+import java.nio.file.Path;
+
+public class IncorrectFormat extends Exception
+{
+    public IncorrectFormat( Path archive, IOException cause )
+    {
+        super( archive.toString(), cause );
+    }
+}

community/dbms/src/main/java/org/neo4j/dbms/archive/Loader.java

@@ -0,0 +1,99 @@
+/*
+ * Copyright (c) 2002-2016 "Neo Technology,"
+ * Network Engine for Objects in Lund AB [http://neotechnology.com]
+ *
+ * This file is part of Neo4j.
+ *
+ * Neo4j is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.neo4j.dbms.archive;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.file.FileAlreadyExistsException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+import org.apache.commons.compress.archivers.ArchiveEntry;
+import org.apache.commons.compress.archivers.ArchiveInputStream;
+import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
+import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
+
+import static java.nio.file.Files.exists;
+
+import static org.neo4j.dbms.archive.Utils.checkWritableDirectory;
+
+public class Loader
+{
+    public void load( Path archive, Path destination ) throws IOException, IncorrectFormat
+    {
+        if ( exists( destination ) )
+        {
+            throw new FileAlreadyExistsException( destination.toString() );
+        }
+        checkWritableDirectory( destination.getParent() );
+        try ( ArchiveInputStream stream = openArchiveIn( archive ) )
+        {
+            ArchiveEntry entry;
+            while ( (entry = nextEntry( stream, archive )) != null )
+            {
+                loadEntry( destination, stream, entry );
+            }
+        }
+    }
+
+    private ArchiveEntry nextEntry( ArchiveInputStream stream, Path archive ) throws IncorrectFormat
+    {
+        try
+        {
+            return stream.getNextEntry();
+        }
+        catch ( IOException e )
+        {
+            throw new IncorrectFormat(archive, e );
+        }
+    }
+
+    private void loadEntry( Path destination, ArchiveInputStream stream, ArchiveEntry entry ) throws IOException
+    {
+        Path file = destination.resolve( entry.getName() );
+        if ( entry.isDirectory() )
+        {
+            Files.createDirectories( file );
+        }
+        else
+        {
+            try ( OutputStream output = Files.newOutputStream( file ) )
+            {
+                Utils.copy( stream, output );
+            }
+        }
+    }
+
+    private static ArchiveInputStream openArchiveIn( Path archive ) throws IOException, IncorrectFormat
+    {
+        InputStream input = Files.newInputStream( archive );
+        GzipCompressorInputStream compressor;
+        try
+        {
+            compressor = new GzipCompressorInputStream( input );
+        }
+        catch ( IOException e )
+        {
+            throw new IncorrectFormat( archive, e );
+        }
+        return new TarArchiveInputStream( compressor );
+    }
+}

community/dbms/src/main/java/org/neo4j/dbms/archive/Utils.java

@@ -0,0 +1,61 @@
+/*
+ * Copyright (c) 2002-2016 "Neo Technology,"
+ * Network Engine for Objects in Lund AB [http://neotechnology.com]
+ *
+ * This file is part of Neo4j.
+ *
+ * Neo4j is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.neo4j.dbms.archive;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.file.AccessDeniedException;
+import java.nio.file.FileSystemException;
+import java.nio.file.NoSuchFileException;
+import java.nio.file.Path;
+
+import static java.nio.file.Files.exists;
+import static java.nio.file.Files.isRegularFile;
+import static java.nio.file.Files.isWritable;
+
+public class Utils
+{
+    public static void checkWritableDirectory( Path directory ) throws FileSystemException
+    {
+        if ( !exists( directory ) )
+        {
+            throw new NoSuchFileException( directory.toString() );
+        }
+        if ( isRegularFile( directory ) )
+        {
+            throw new FileSystemException( directory.toString() + ": Not a directory" );
+        }
+        if ( !isWritable( directory ) )
+        {
+            throw new AccessDeniedException( directory.toString() );
+        }
+    }
+
+    public static void copy( InputStream in, OutputStream out ) throws IOException
+    {
+        final byte[] buffer = new byte[8192];
+        int n;
+        while ( -1 != (n = in.read( buffer )) )
+        {
+            out.write( buffer, 0, n );
+        }
+    }
+}