Fix the path mapped migrator checksum issue

deployments/pathmap-migrator/src/main/java/org/commonjava/indy/pathmap/migrate/CassandraMigrator.java

@@ -28,6 +28,7 @@
 import java.io.FileInputStream;
 import java.io.IOException;
 import java.nio.file.Paths;
+import java.security.NoSuchAlgorithmException;
 import java.util.Date;
 import java.util.Map;
 
@@ -41,29 +42,32 @@ public class CassandraMigrator
 
     private final IndyStoreBasedPathGenerator storePathGen;
 
-    private final ChecksumCalculator checksumCalculator;
-
     private final String baseDir;
 
+    private final boolean dedup;
+
+    private final String dedupAlgo;
+
     private CassandraMigrator( final PathMappedStorageConfig config, final String baseDir,
-                               final ChecksumCalculator checksumCalculator )
+                               final boolean dedup, final String dedupAlgo )
     {
         this.pathDB = new CassandraPathDB( config );
         this.storePathGen = new IndyStoreBasedPathGenerator( baseDir );
         this.physicalStore = new FileBasedPhysicalStore( new File( baseDir ) );
-        this.checksumCalculator = checksumCalculator;
+        this.dedup = dedup;
+        this.dedupAlgo = dedupAlgo;
         this.baseDir = baseDir;
     }
 
     public static CassandraMigrator getMigrator( final Map<String, Object> cassandraConfig, final String baseDir,
-                                          final ChecksumCalculator checksumCalculator )
+                                                 final boolean dedup, final String dedupAlgo )
     {
         synchronized ( CassandraMigrator.class )
         {
             if ( migrator == null )
             {
                 final PathMappedStorageConfig config = new DefaultPathMappedStorageConfig( cassandraConfig );
-                migrator = new CassandraMigrator( config, baseDir, checksumCalculator );
+                migrator = new CassandraMigrator( config, baseDir, dedup, dedupAlgo );
             }
         }
         return migrator;
@@ -86,17 +90,19 @@ public void migrate( final String physicalFilePath )
                                         physicalFilePath );
         }
 
-        final String checksum;
-        try
-        {
-            checksum = calculateChecksum( file );
-        }
-        catch ( IOException e )
+        String checksum = null;
+        if ( dedup )
         {
-            throw new MigrateException(
-                    String.format( "Error: Can not get file checksum for file of %s", physicalFilePath ), e );
+            try
+            {
+                checksum = calculateChecksum( file );
+            }
+            catch ( IOException e )
+            {
+                throw new MigrateException(
+                                String.format( "Error: Can not get file checksum for file of %s", physicalFilePath ), e );
+            }
         }
-
         final String fileSystem = storePathGen.generateFileSystem( physicalFilePath );
         final String path = storePathGen.generatePath( physicalFilePath );
         final String storePath = storePathGen.generateStorePath( physicalFilePath );
@@ -115,16 +121,24 @@ public void migrate( final String physicalFilePath )
     private String calculateChecksum( File file )
             throws IOException
     {
-        if ( checksumCalculator == null )
-        {
-            return null;
-        }
-
         if ( !file.exists() || !file.isFile() )
         {
             throw new IOException(
                     String.format( "Digest error: file not exists or not a regular file for file %s", file ) );
         }
+
+        ChecksumCalculator checksumCalculator = null;
+        {
+            try
+            {
+                checksumCalculator = new ChecksumCalculator( dedupAlgo );
+            }
+            catch ( NoSuchAlgorithmException e )
+            {
+                // verified, ex never happen
+            }
+        }
+
         try (FileInputStream is = new FileInputStream( file ))
         {
             checksumCalculator.update( IOUtils.toByteArray( is ) );

deployments/pathmap-migrator/src/main/java/org/commonjava/indy/pathmap/migrate/MigrateCmd.java

@@ -111,8 +111,11 @@ public void run( final MigrateOptions options )
             }
         }
 
+        stop( options );
     }
 
+    private Timer progressTimer = new Timer();
+
     private void init( MigrateOptions options )
     {
         // Reload last processed paths count
@@ -142,7 +145,13 @@ private void init( MigrateOptions options )
 
         final long period = 15000L;
         // Trigger progress update task.
-        new Timer().schedule( new UpdateProgressTask( options ), period, period );
+        progressTimer.schedule( new UpdateProgressTask( options ), period, period );
+    }
+
+    private void stop( MigrateOptions options )
+    {
+        new UpdateProgressTask( options ).run(); // last run
+        progressTimer.cancel();
     }
 
     private void storeFailedPaths( MigrateOptions options, List<String> failedPaths )

deployments/pathmap-migrator/src/main/java/org/commonjava/indy/pathmap/migrate/MigrateOptions.java

@@ -411,20 +411,19 @@ private void initMigrator()
                 cassandraProps.put( PROP_CASSANDRA_PASS, getCassandraPass() );
             }
 
-            ChecksumCalculator calculator = null;
             if ( isDedupe() )
             {
                 try
                 {
-                    calculator = new ChecksumCalculator( getDedupeAlgorithm() );
+                    new ChecksumCalculator( getDedupeAlgorithm() ); // verify algo
                 }
                 catch ( NoSuchAlgorithmException e )
                 {
                     throw new MigrateException(
                             String.format( "Error: checksum algorithm not supported: %s", getDedupeAlgorithm() ), e );
                 }
             }
-            migrator = CassandraMigrator.getMigrator( cassandraProps, getBaseDir(), calculator );
+            migrator = CassandraMigrator.getMigrator( cassandraProps, getBaseDir(), isDedupe(), getDedupeAlgorithm() );
         }
     }
 

deployments/pathmap-migrator/src/test/java/org/commonjava/indy/pathmap/migrate/ChecksumCalculatorTest.java

@@ -0,0 +1,97 @@
+/**
+ * Copyright (C) 2013~2019 Red Hat, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.commonjava.indy.pathmap.migrate;
+
+import org.apache.commons.io.IOUtils;
+import org.commonjava.storage.pathmapped.util.ChecksumCalculator;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
+
+public class ChecksumCalculatorTest
+{
+    @Rule
+    public TemporaryFolder temporaryFolder = new TemporaryFolder();
+
+    private String dupContent = "abc";
+
+    private String otherContent = "def";
+
+    private File f1, f2, f3;
+
+    @Before
+    public void prepare() throws Exception
+    {
+        f1 = temporaryFolder.newFile( "f1" );
+        try (FileOutputStream out = new FileOutputStream( f1 ))
+        {
+            IOUtils.write( dupContent, out );
+        }
+        f2 = temporaryFolder.newFile( "f2" );
+        try (FileOutputStream out = new FileOutputStream( f2 ))
+        {
+            IOUtils.write( dupContent, out );
+        }
+        f3 = temporaryFolder.newFile( "f3" );
+        try (FileOutputStream out = new FileOutputStream( f3 ))
+        {
+            IOUtils.write( otherContent, out );
+        }
+    }
+
+    @Ignore
+    @Test
+    public void run() throws Exception
+    {
+        String s1 = calculateChecksum( f1 );
+        System.out.println( ">>> s1=" + s1 );
+
+        String s2 = calculateChecksum( f2 );
+        System.out.println( ">>> s2=" + s2 );
+
+        String s3 = calculateChecksum( f3 );
+        System.out.println( ">>> s3=" + s3 );
+
+        assertEquals( s1, s2 );
+        assertNotEquals( s1, s3 );
+    }
+
+    private String calculateChecksum( File file ) throws Exception
+    {
+        ChecksumCalculator checksumCalculator = new ChecksumCalculator( "MD5" );
+
+        if ( !file.exists() || !file.isFile() )
+        {
+            throw new IOException(
+                            String.format( "Digest error: file not exists or not a regular file for file %s", file ) );
+        }
+        try (FileInputStream is = new FileInputStream( file ))
+        {
+            checksumCalculator.update( IOUtils.toByteArray( is ) );
+        }
+        return checksumCalculator.getDigestHex();
+    }
+}