Add progress output to neo4j-admin dump.

community/dbms/src/main/java/org/neo4j/commandline/dbms/DumpCommandProvider.java

@@ -69,6 +69,6 @@ public AdminCommandSection commandSection()
     @Nonnull
     public AdminCommand create( Path homeDir, Path configDir, OutsideWorld outsideWorld )
     {
-        return new DumpCommand( homeDir, configDir, new Dumper() );
+        return new DumpCommand( homeDir, configDir, new Dumper( outsideWorld.errorStream() ) );
     }
 }

community/dbms/src/main/java/org/neo4j/dbms/archive/Dumper.java

@@ -26,13 +26,21 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
+import java.io.PrintStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.StandardOpenOption;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
 import java.util.function.Predicate;
 
 import org.neo4j.commandline.Util;
 import org.neo4j.function.ThrowingAction;
+import org.neo4j.util.VisibleForTesting;
 
 import static org.neo4j.dbms.archive.Utils.checkWritableDirectory;
 import static org.neo4j.dbms.archive.Utils.copy;
@@ -46,17 +54,66 @@
 
 public class Dumper
 {
+    private final List<ArchiveOperation> operations;
+    private final ProgressPrinter progressPrinter;
+
+    @VisibleForTesting
+    Dumper()
+    {
+        operations = new ArrayList<>();
+        progressPrinter = new ProgressPrinter( null );
+    }
+
+    public Dumper( PrintStream output )
+    {
+        operations = new ArrayList<>();
+        progressPrinter = new ProgressPrinter( output );
+    }
+
     public void dump( Path dbPath, Path transactionalLogsPath, Path archive, CompressionFormat format, Predicate<Path> exclude )
             throws IOException
     {
         checkWritableDirectory( archive.getParent() );
         try ( ArchiveOutputStream stream = openArchiveOut( archive, format ) )
         {
+            operations.clear();
+
             visitPath( dbPath, exclude, stream );
             if ( !Util.isSameOrChildPath( dbPath, transactionalLogsPath ) )
             {
                 visitPath( transactionalLogsPath, exclude, stream );
             }
+
+            progressPrinter.reset();
+            for ( ArchiveOperation operation : operations )
+            {
+                progressPrinter.maxBytes += operation.size;
+                progressPrinter.maxFiles += operation.isFile ? 1 : 0;
+            }
+
+            ScheduledExecutorService timer = Executors.newSingleThreadScheduledExecutor();
+            ScheduledFuture<?> timerFuture = timer.scheduleAtFixedRate( progressPrinter::printOnNextUpdate, 0, 100, TimeUnit.MILLISECONDS );
+            try
+            {
+                for ( ArchiveOperation operation : operations )
+                {
+                    operation.addToArchive();
+                }
+            }
+            finally
+            {
+                timerFuture.cancel( false );
+                timer.shutdown();
+                try
+                {
+                    timer.awaitTermination( 10, TimeUnit.SECONDS );
+                }
+                catch ( InterruptedException ignore )
+                {
+                }
+            }
+            progressPrinter.done();
+            progressPrinter.printProgress();
         }
     }
 
@@ -97,22 +154,47 @@ private void dumpDirectory( Path root, ArchiveOutputStream stream, Path dir ) th
     private void withEntry( ThrowingAction<IOException> operation, Path root, ArchiveOutputStream stream, Path file )
             throws IOException
     {
-        ArchiveEntry entry = createEntry( file, root, stream );
-        stream.putArchiveEntry( entry );
-        operation.apply();
-        stream.closeArchiveEntry();
+        operations.add( new ArchiveOperation( operation, root, stream, file ) );
     }
 
-    private ArchiveEntry createEntry( Path file, Path root, ArchiveOutputStream archive ) throws IOException
+    private void writeFile( Path file, ArchiveOutputStream archiveStream ) throws IOException
     {
-        return archive.createArchiveEntry( file.toFile(), "./" + root.relativize( file ).toString() );
+        try ( InputStream in = Files.newInputStream( file ) )
+        {
+            copy( in, archiveStream, progressPrinter );
+        }
     }
 
-    private void writeFile( Path file, ArchiveOutputStream archiveStream ) throws IOException
+    private static class ArchiveOperation
     {
-        try ( InputStream in = Files.newInputStream( file ) )
+        final ThrowingAction<IOException> operation;
+        final long size;
+        final boolean isFile;
+        final Path root;
+        final ArchiveOutputStream stream;
+        final Path file;
+
+        private ArchiveOperation( ThrowingAction<IOException> operation, Path root, ArchiveOutputStream stream, Path file ) throws IOException
+        {
+            this.operation = operation;
+            this.isFile = Files.isRegularFile( file );
+            this.size = isFile ? Files.size( file ) : 0;
+            this.root = root;
+            this.stream = stream;
+            this.file = file;
+        }
+
+        void addToArchive() throws IOException
+        {
+            ArchiveEntry entry = createEntry( file, root, stream );
+            stream.putArchiveEntry( entry );
+            operation.apply();
+            stream.closeArchiveEntry();
+        }
+
+        private ArchiveEntry createEntry( Path file, Path root, ArchiveOutputStream archive ) throws IOException
         {
-            copy( in, archiveStream );
+            return archive.createArchiveEntry( file.toFile(), "./" + root.relativize( file ).toString() );
         }
     }
 }

community/dbms/src/main/java/org/neo4j/dbms/archive/Loader.java

@@ -112,7 +112,7 @@ private void loadEntry( Path destination, ArchiveInputStream stream, ArchiveEntr
         {
             try ( OutputStream output = Files.newOutputStream( file ) )
             {
-                Utils.copy( stream, output );
+                Utils.copy( stream, output, new ProgressPrinter( null ) );
             }
         }
     }

community/dbms/src/main/java/org/neo4j/dbms/archive/ProgressPrinter.java

@@ -0,0 +1,96 @@
+/*
+ * Copyright (c) 2002-2019 "Neo4j,"
+ * Neo4j Sweden AB [http://neo4j.com]
+ *
+ * This file is part of Neo4j.
+ *
+ * Neo4j is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.neo4j.dbms.archive;
+
+import java.io.PrintStream;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.neo4j.io.ByteUnit;
+
+class ProgressPrinter
+{
+    private final AtomicBoolean printUpdate;
+    private final PrintStream output;
+    long maxBytes;
+    long maxFiles;
+    long currentBytes;
+    long currentFiles;
+    boolean done;
+
+    ProgressPrinter( PrintStream output )
+    {
+        this.output = output;
+        printUpdate = new AtomicBoolean();
+    }
+
+    void printOnNextUpdate()
+    {
+        printUpdate.set( true );
+    }
+
+    void reset()
+    {
+        maxBytes = 0;
+        maxFiles = 0;
+        currentBytes = 0;
+        currentFiles = 0;
+    }
+
+    void beginFile()
+    {
+        currentFiles++;
+    }
+
+    void addBytes( long n )
+    {
+        currentBytes += n;
+        if ( printUpdate.get() )
+        {
+            printProgress();
+            printUpdate.set( false );
+        }
+    }
+
+    void endFile()
+    {
+        printProgress();
+    }
+
+    void done()
+    {
+        done = true;
+    }
+
+    void printProgress()
+    {
+        if ( output != null )
+        {
+            if ( done )
+            {
+                output.println( "\rDone: " + currentFiles + " files, " + ByteUnit.bytesToString( currentBytes ) + " processed." );
+            }
+            else
+            {
+                double progress = (currentBytes / (double) maxBytes) * 100;
+                output.print( "\rFiles: " + currentFiles + '/' + maxFiles + ", data: " + String.format( "%4.1f%%", progress ));
+            }
+        }
+    }
+}

community/dbms/src/main/java/org/neo4j/dbms/archive/Utils.java

@@ -53,13 +53,16 @@ public static void checkWritableDirectory( Path directory ) throws FileSystemExc
         }
     }
 
-    public static void copy( InputStream in, OutputStream out ) throws IOException
+    public static void copy( InputStream in, OutputStream out, ProgressPrinter progressPrinter ) throws IOException
     {
+        progressPrinter.beginFile();
         final byte[] buffer = new byte[8192];
         int n;
         while ( -1 != (n = in.read( buffer )) )
         {
             out.write( buffer, 0, n );
+            progressPrinter.addBytes( n );
         }
+        progressPrinter.endFile();
     }
 }

community/dbms/src/test/java/org/neo4j/commandline/dbms/DumpCommandTest.java

@@ -363,9 +363,8 @@ void shouldPrintNiceHelp() throws Exception
                             "%n" +
                             "options:%n" +
                             "  --database=<name>         Name of database. [default:" + GraphDatabaseSettings.DEFAULT_DATABASE_NAME + "]%n" +
-                            "  --zstd=<true|false>       Use Zstandard compression (the dump file will not be%n" +
-                            "                            loadable by older versions of Neo4j).%n" +
-                            "                            [default:false]%n" +
+                            "  --zstd=<true|false>       Use Zstandard compression (might not be loadable by%n" +
+                            "                            older versions of Neo4j). [default:false]%n" +
                             "  --to=<destination-path>   Destination (file or folder) of database dump.%n" ),
                     baos.toString() );
         }

community/dbms/src/test/java/org/neo4j/dbms/archive/ProgressPrinterTest.java

@@ -0,0 +1,68 @@
+/*
+ * Copyright (c) 2002-2019 "Neo4j,"
+ * Neo4j Sweden AB [http://neo4j.com]
+ *
+ * This file is part of Neo4j.
+ *
+ * Neo4j is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.neo4j.dbms.archive;
+
+import org.junit.jupiter.api.Test;
+
+import java.io.ByteArrayOutputStream;
+import java.io.PrintStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+class ProgressPrinterTest
+{
+    @Test
+    void progressOutput()
+    {
+        ByteArrayOutputStream bout = new ByteArrayOutputStream();
+        PrintStream printStream = new PrintStream( bout );
+        ProgressPrinter progressPrinter = new ProgressPrinter( printStream );
+        progressPrinter.maxBytes = 1000;
+        progressPrinter.maxFiles = 10;
+
+        progressPrinter.beginFile();
+        progressPrinter.addBytes( 5 );
+        progressPrinter.endFile();
+        progressPrinter.beginFile();
+        progressPrinter.addBytes( 50 );
+        progressPrinter.addBytes( 50 );
+        progressPrinter.printOnNextUpdate();
+        progressPrinter.addBytes( 100 );
+        progressPrinter.endFile();
+        progressPrinter.beginFile();
+        progressPrinter.printOnNextUpdate();
+        progressPrinter.addBytes( 100 );
+        progressPrinter.endFile();
+        progressPrinter.done();
+        progressPrinter.printProgress();
+
+        printStream.flush();
+        String output = bout.toString();
+        assertEquals( output,
+                "\rFiles: 1/10, data:  0.5%" +
+                "\rFiles: 2/10, data: 20.5%" +
+                "\rFiles: 2/10, data: 20.5%" +
+                "\rFiles: 3/10, data: 30.5%" +
+                "\rFiles: 3/10, data: 30.5%" +
+                "\rDone: 3 files, 305B processed." +
+                        System.lineSeparator()
+        );
+    }
+}