From 8b9f34b92582dbc485f4d6a7fc17bf8ef6bf32b7 Mon Sep 17 00:00:00 2001 From: Rui Han Date: Thu, 14 Nov 2019 11:57:16 +0800 Subject: [PATCH] Fix the path mapped migrator checksum issue --- .../pathmap/migrate/CassandraMigrator.java | 54 +++++++---- .../indy/pathmap/migrate/MigrateCmd.java | 11 ++- .../indy/pathmap/migrate/MigrateOptions.java | 5 +- .../migrate/ChecksumCalculatorTest.java | 97 +++++++++++++++++++ 4 files changed, 143 insertions(+), 24 deletions(-) create mode 100644 deployments/pathmap-migrator/src/test/java/org/commonjava/indy/pathmap/migrate/ChecksumCalculatorTest.java diff --git a/deployments/pathmap-migrator/src/main/java/org/commonjava/indy/pathmap/migrate/CassandraMigrator.java b/deployments/pathmap-migrator/src/main/java/org/commonjava/indy/pathmap/migrate/CassandraMigrator.java index 7ae7bf0158..65a042fe83 100644 --- a/deployments/pathmap-migrator/src/main/java/org/commonjava/indy/pathmap/migrate/CassandraMigrator.java +++ b/deployments/pathmap-migrator/src/main/java/org/commonjava/indy/pathmap/migrate/CassandraMigrator.java @@ -28,6 +28,7 @@ import java.io.FileInputStream; import java.io.IOException; import java.nio.file.Paths; +import java.security.NoSuchAlgorithmException; import java.util.Date; import java.util.Map; @@ -41,29 +42,32 @@ public class CassandraMigrator private final IndyStoreBasedPathGenerator storePathGen; - private final ChecksumCalculator checksumCalculator; - private final String baseDir; + private final boolean dedup; + + private final String dedupAlgo; + private CassandraMigrator( final PathMappedStorageConfig config, final String baseDir, - final ChecksumCalculator checksumCalculator ) + final boolean dedup, final String dedupAlgo ) { this.pathDB = new CassandraPathDB( config ); this.storePathGen = new IndyStoreBasedPathGenerator( baseDir ); this.physicalStore = new FileBasedPhysicalStore( new File( baseDir ) ); - this.checksumCalculator = checksumCalculator; + this.dedup = dedup; + this.dedupAlgo = dedupAlgo; this.baseDir = baseDir; } public static CassandraMigrator getMigrator( final Map cassandraConfig, final String baseDir, - final ChecksumCalculator checksumCalculator ) + final boolean dedup, final String dedupAlgo ) { synchronized ( CassandraMigrator.class ) { if ( migrator == null ) { final PathMappedStorageConfig config = new DefaultPathMappedStorageConfig( cassandraConfig ); - migrator = new CassandraMigrator( config, baseDir, checksumCalculator ); + migrator = new CassandraMigrator( config, baseDir, dedup, dedupAlgo ); } } return migrator; @@ -86,17 +90,19 @@ public void migrate( final String physicalFilePath ) physicalFilePath ); } - final String checksum; - try - { - checksum = calculateChecksum( file ); - } - catch ( IOException e ) + String checksum = null; + if ( dedup ) { - throw new MigrateException( - String.format( "Error: Can not get file checksum for file of %s", physicalFilePath ), e ); + try + { + checksum = calculateChecksum( file ); + } + catch ( IOException e ) + { + throw new MigrateException( + String.format( "Error: Can not get file checksum for file of %s", physicalFilePath ), e ); + } } - final String fileSystem = storePathGen.generateFileSystem( physicalFilePath ); final String path = storePathGen.generatePath( physicalFilePath ); final String storePath = storePathGen.generateStorePath( physicalFilePath ); @@ -115,16 +121,24 @@ public void migrate( final String physicalFilePath ) private String calculateChecksum( File file ) throws IOException { - if ( checksumCalculator == null ) - { - return null; - } - if ( !file.exists() || !file.isFile() ) { throw new IOException( String.format( "Digest error: file not exists or not a regular file for file %s", file ) ); } + + ChecksumCalculator checksumCalculator = null; + { + try + { + checksumCalculator = new ChecksumCalculator( dedupAlgo ); + } + catch ( NoSuchAlgorithmException e ) + { + // verified, ex never happen + } + } + try (FileInputStream is = new FileInputStream( file )) { checksumCalculator.update( IOUtils.toByteArray( is ) ); diff --git a/deployments/pathmap-migrator/src/main/java/org/commonjava/indy/pathmap/migrate/MigrateCmd.java b/deployments/pathmap-migrator/src/main/java/org/commonjava/indy/pathmap/migrate/MigrateCmd.java index 639773fb2b..11608f4675 100644 --- a/deployments/pathmap-migrator/src/main/java/org/commonjava/indy/pathmap/migrate/MigrateCmd.java +++ b/deployments/pathmap-migrator/src/main/java/org/commonjava/indy/pathmap/migrate/MigrateCmd.java @@ -111,8 +111,11 @@ public void run( final MigrateOptions options ) } } + stop( options ); } + private Timer progressTimer = new Timer(); + private void init( MigrateOptions options ) { // Reload last processed paths count @@ -142,7 +145,13 @@ private void init( MigrateOptions options ) final long period = 15000L; // Trigger progress update task. - new Timer().schedule( new UpdateProgressTask( options ), period, period ); + progressTimer.schedule( new UpdateProgressTask( options ), period, period ); + } + + private void stop( MigrateOptions options ) + { + new UpdateProgressTask( options ).run(); // last run + progressTimer.cancel(); } private void storeFailedPaths( MigrateOptions options, List failedPaths ) diff --git a/deployments/pathmap-migrator/src/main/java/org/commonjava/indy/pathmap/migrate/MigrateOptions.java b/deployments/pathmap-migrator/src/main/java/org/commonjava/indy/pathmap/migrate/MigrateOptions.java index 00192c73ea..a11f06b693 100644 --- a/deployments/pathmap-migrator/src/main/java/org/commonjava/indy/pathmap/migrate/MigrateOptions.java +++ b/deployments/pathmap-migrator/src/main/java/org/commonjava/indy/pathmap/migrate/MigrateOptions.java @@ -411,12 +411,11 @@ private void initMigrator() cassandraProps.put( PROP_CASSANDRA_PASS, getCassandraPass() ); } - ChecksumCalculator calculator = null; if ( isDedupe() ) { try { - calculator = new ChecksumCalculator( getDedupeAlgorithm() ); + new ChecksumCalculator( getDedupeAlgorithm() ); // verify algo } catch ( NoSuchAlgorithmException e ) { @@ -424,7 +423,7 @@ private void initMigrator() String.format( "Error: checksum algorithm not supported: %s", getDedupeAlgorithm() ), e ); } } - migrator = CassandraMigrator.getMigrator( cassandraProps, getBaseDir(), calculator ); + migrator = CassandraMigrator.getMigrator( cassandraProps, getBaseDir(), isDedupe(), getDedupeAlgorithm() ); } } diff --git a/deployments/pathmap-migrator/src/test/java/org/commonjava/indy/pathmap/migrate/ChecksumCalculatorTest.java b/deployments/pathmap-migrator/src/test/java/org/commonjava/indy/pathmap/migrate/ChecksumCalculatorTest.java new file mode 100644 index 0000000000..edf51f2508 --- /dev/null +++ b/deployments/pathmap-migrator/src/test/java/org/commonjava/indy/pathmap/migrate/ChecksumCalculatorTest.java @@ -0,0 +1,97 @@ +/** + * Copyright (C) 2013~2019 Red Hat, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.commonjava.indy.pathmap.migrate; + +import org.apache.commons.io.IOUtils; +import org.commonjava.storage.pathmapped.util.ChecksumCalculator; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; + +public class ChecksumCalculatorTest +{ + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + private String dupContent = "abc"; + + private String otherContent = "def"; + + private File f1, f2, f3; + + @Before + public void prepare() throws Exception + { + f1 = temporaryFolder.newFile( "f1" ); + try (FileOutputStream out = new FileOutputStream( f1 )) + { + IOUtils.write( dupContent, out ); + } + f2 = temporaryFolder.newFile( "f2" ); + try (FileOutputStream out = new FileOutputStream( f2 )) + { + IOUtils.write( dupContent, out ); + } + f3 = temporaryFolder.newFile( "f3" ); + try (FileOutputStream out = new FileOutputStream( f3 )) + { + IOUtils.write( otherContent, out ); + } + } + + @Ignore + @Test + public void run() throws Exception + { + String s1 = calculateChecksum( f1 ); + System.out.println( ">>> s1=" + s1 ); + + String s2 = calculateChecksum( f2 ); + System.out.println( ">>> s2=" + s2 ); + + String s3 = calculateChecksum( f3 ); + System.out.println( ">>> s3=" + s3 ); + + assertEquals( s1, s2 ); + assertNotEquals( s1, s3 ); + } + + private String calculateChecksum( File file ) throws Exception + { + ChecksumCalculator checksumCalculator = new ChecksumCalculator( "MD5" ); + + if ( !file.exists() || !file.isFile() ) + { + throw new IOException( + String.format( "Digest error: file not exists or not a regular file for file %s", file ) ); + } + try (FileInputStream is = new FileInputStream( file )) + { + checksumCalculator.update( IOUtils.toByteArray( is ) ); + } + return checksumCalculator.getDigestHex(); + } +}