Permalink
Browse files

Merge pull request #838 from sonatype/NEXUS-5094_purgeSnapshotWithCla…

…ssifier

REVIEW [NEXUS-5094] Purging snapshot task should take into account classifier
  • Loading branch information...
2 parents 72eb270 + 1418fc8 commit c5cbb6c6a8ab75177f4b04b970f90b81af4cfc46 @kellyrob99 kellyrob99 committed May 1, 2013
Showing with 829 additions and 59 deletions.
  1. +132 −59 nexus-core/src/main/java/org/sonatype/nexus/maven/tasks/DefaultSnapshotRemover.java
  2. +33 −0 nexus-core/src/test/java/org/sonatype/nexus/maven/tasks/DefaultSnapshotRemoverIT.java
  3. +32 −0 nexus-core/src/test/resources/reposes/snapshots/org/sonatype/test/1.0-SNAPSHOT/maven-metadata.xml
  4. +1 −0 ...s-core/src/test/resources/reposes/snapshots/org/sonatype/test/1.0-SNAPSHOT/maven-metadata.xml.md5
  5. +1 −0 ...-core/src/test/resources/reposes/snapshots/org/sonatype/test/1.0-SNAPSHOT/maven-metadata.xml.sha1
  6. BIN ...t/resources/reposes/snapshots/org/sonatype/test/1.0-SNAPSHOT/test-1.0-20130501.154949-1-alpha.jar
  7. +1 −0 ...sources/reposes/snapshots/org/sonatype/test/1.0-SNAPSHOT/test-1.0-20130501.154949-1-alpha.jar.md5
  8. +1 −0 ...ources/reposes/snapshots/org/sonatype/test/1.0-SNAPSHOT/test-1.0-20130501.154949-1-alpha.jar.sha1
  9. +122 −0 ...rc/test/resources/reposes/snapshots/org/sonatype/test/1.0-SNAPSHOT/test-1.0-20130501.154949-1.pom
  10. +1 −0 ...est/resources/reposes/snapshots/org/sonatype/test/1.0-SNAPSHOT/test-1.0-20130501.154949-1.pom.md5
  11. +1 −0 ...st/resources/reposes/snapshots/org/sonatype/test/1.0-SNAPSHOT/test-1.0-20130501.154949-1.pom.sha1
  12. BIN ...t/resources/reposes/snapshots/org/sonatype/test/1.0-SNAPSHOT/test-1.0-20130501.154953-2-alpha.jar
  13. +1 −0 ...sources/reposes/snapshots/org/sonatype/test/1.0-SNAPSHOT/test-1.0-20130501.154953-2-alpha.jar.md5
  14. +1 −0 ...ources/reposes/snapshots/org/sonatype/test/1.0-SNAPSHOT/test-1.0-20130501.154953-2-alpha.jar.sha1
  15. +122 −0 ...rc/test/resources/reposes/snapshots/org/sonatype/test/1.0-SNAPSHOT/test-1.0-20130501.154953-2.pom
  16. +1 −0 ...est/resources/reposes/snapshots/org/sonatype/test/1.0-SNAPSHOT/test-1.0-20130501.154953-2.pom.md5
  17. +1 −0 ...st/resources/reposes/snapshots/org/sonatype/test/1.0-SNAPSHOT/test-1.0-20130501.154953-2.pom.sha1
  18. BIN ...t/resources/reposes/snapshots/org/sonatype/test/1.0-SNAPSHOT/test-1.0-20130501.154956-3-alpha.jar
  19. +1 −0 ...sources/reposes/snapshots/org/sonatype/test/1.0-SNAPSHOT/test-1.0-20130501.154956-3-alpha.jar.md5
  20. +1 −0 ...ources/reposes/snapshots/org/sonatype/test/1.0-SNAPSHOT/test-1.0-20130501.154956-3-alpha.jar.sha1
  21. +122 −0 ...rc/test/resources/reposes/snapshots/org/sonatype/test/1.0-SNAPSHOT/test-1.0-20130501.154956-3.pom
  22. +1 −0 ...est/resources/reposes/snapshots/org/sonatype/test/1.0-SNAPSHOT/test-1.0-20130501.154956-3.pom.md5
  23. +1 −0 ...st/resources/reposes/snapshots/org/sonatype/test/1.0-SNAPSHOT/test-1.0-20130501.154956-3.pom.sha1
  24. BIN ...st/resources/reposes/snapshots/org/sonatype/test/1.0-SNAPSHOT/test-1.0-20130501.155001-4-beta.jar
  25. +1 −0 ...esources/reposes/snapshots/org/sonatype/test/1.0-SNAPSHOT/test-1.0-20130501.155001-4-beta.jar.md5
  26. +1 −0 ...sources/reposes/snapshots/org/sonatype/test/1.0-SNAPSHOT/test-1.0-20130501.155001-4-beta.jar.sha1
  27. +122 −0 ...rc/test/resources/reposes/snapshots/org/sonatype/test/1.0-SNAPSHOT/test-1.0-20130501.155001-4.pom
  28. +1 −0 ...est/resources/reposes/snapshots/org/sonatype/test/1.0-SNAPSHOT/test-1.0-20130501.155001-4.pom.md5
  29. +1 −0 ...st/resources/reposes/snapshots/org/sonatype/test/1.0-SNAPSHOT/test-1.0-20130501.155001-4.pom.sha1
  30. BIN ...st/resources/reposes/snapshots/org/sonatype/test/1.0-SNAPSHOT/test-1.0-20130501.155005-5-beta.jar
  31. +1 −0 ...esources/reposes/snapshots/org/sonatype/test/1.0-SNAPSHOT/test-1.0-20130501.155005-5-beta.jar.md5
  32. +1 −0 ...sources/reposes/snapshots/org/sonatype/test/1.0-SNAPSHOT/test-1.0-20130501.155005-5-beta.jar.sha1
  33. +122 −0 ...rc/test/resources/reposes/snapshots/org/sonatype/test/1.0-SNAPSHOT/test-1.0-20130501.155005-5.pom
  34. +1 −0 ...est/resources/reposes/snapshots/org/sonatype/test/1.0-SNAPSHOT/test-1.0-20130501.155005-5.pom.md5
  35. +1 −0 ...st/resources/reposes/snapshots/org/sonatype/test/1.0-SNAPSHOT/test-1.0-20130501.155005-5.pom.sha1
@@ -13,10 +13,10 @@
package org.sonatype.nexus.maven.tasks;
import static com.google.common.base.Preconditions.checkNotNull;
+import static org.apache.commons.lang.StringUtils.*;
import java.util.ArrayList;
import java.util.Collection;
-import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -27,6 +27,7 @@
import javax.inject.Named;
import javax.inject.Singleton;
+import com.google.common.collect.Maps;
import org.sonatype.nexus.logging.AbstractLoggingComponent;
import org.sonatype.nexus.proxy.IllegalOperationException;
import org.sonatype.nexus.proxy.ItemNotFoundException;
@@ -482,7 +483,7 @@ public void doOnCollectionExit( WalkerContext context, StorageCollectionItem col
}
else
{
- getLogger().debug( "itemTimestamp=" + itemTimestamp + ", dateTreshold=" + dateThreshold );
+ getLogger().debug( "itemTimestamp=" + itemTimestamp + ", dateThreshold=" + dateThreshold );
// if dateThreshold is not used (zero days) OR
// if itemTimestamp is less then dateThreshold (NB: both are positive!)
@@ -494,6 +495,7 @@ public void doOnCollectionExit( WalkerContext context, StorageCollectionItem col
}
else
{
+ //do not delete if dateThreshold not met
addStorageFileItemToMap( remainingSnapshotsAndFiles, gav, (StorageFileItem) item );
}
}
@@ -543,89 +545,160 @@ public void doOnCollectionExit( WalkerContext context, StorageCollectionItem col
else
{
// and now check some things
- if ( remainingSnapshotsAndFiles.size() < request.getMinCountOfSnapshotsToKeep() )
+
+ Map<String, Map<Version, List<StorageFileItem>>> remainingByClassifier =
+ partitionFiles( remainingSnapshotsAndFiles );
+
+ Map<String, Map<Version, List<StorageFileItem>>> deletableByClassifier =
+ partitionFiles( deletableSnapshotsAndFiles );
+
+ for ( String classifier : deletableByClassifier.keySet() )
{
- // do something
- if ( remainingSnapshotsAndFiles.size() + deletableSnapshotsAndFiles.size() < request.getMinCountOfSnapshotsToKeep() )
+ getLogger().debug( "Processing deletes for classifier: {}", isNotEmpty(classifier) ? classifier : "-NONE-" );
+ //if there are no files remaining for a classifier, introduce an empty map for processing
+ if(!remainingByClassifier.containsKey( classifier ))
{
- // delete nothing, since there is less snapshots in total as allowed
- deletableSnapshotsAndFiles.clear();
+ remainingByClassifier.put( classifier, Maps.<Version, List<StorageFileItem>>newHashMap() );
}
- else
- {
- TreeSet<Version> keys = new TreeSet<Version>( deletableSnapshotsAndFiles.keySet() );
+ reconcileDeletableFilesWithMinCount( remainingByClassifier.get( classifier ),
+ deletableByClassifier.get( classifier ),
+ request.getMinCountOfSnapshotsToKeep() );
+ deleteFiles( context, deletableByClassifier.get( classifier ), remainingByClassifier.get( classifier ).size() );
+ }
+ }
- while ( !keys.isEmpty()
- && remainingSnapshotsAndFiles.size() < request.getMinCountOfSnapshotsToKeep() )
- {
- Version keyToMove = keys.last();
+ removeDirectoryIfEmpty( repository, coll );
- if ( remainingSnapshotsAndFiles.containsKey( keyToMove ) )
- {
- remainingSnapshotsAndFiles.get( keyToMove ).addAll(
- deletableSnapshotsAndFiles.get( keyToMove ) );
- }
- else
- {
- remainingSnapshotsAndFiles.put( keyToMove, deletableSnapshotsAndFiles.get( keyToMove ) );
- }
+ updateMetadataIfNecessary( context, coll );
+
+ }
+
+ private void reconcileDeletableFilesWithMinCount(
+ Map<Version, List<StorageFileItem>> remainingSnapshotsAndFiles,
+ Map<Version, List<StorageFileItem>> deletableSnapshotsAndFiles, int minCount )
+ {
+ if ( remainingSnapshotsAndFiles.size() < minCount )
+ {
+ // do something
+ if ( remainingSnapshotsAndFiles.size() + deletableSnapshotsAndFiles.size() < minCount )
+ {
+ // delete nothing, since there is less snapshots in total as allowed
+ deletableSnapshotsAndFiles.clear();
+ }
+ else
+ {
+ TreeSet<Version> keys = new TreeSet<Version>( deletableSnapshotsAndFiles.keySet() );
- deletableSnapshotsAndFiles.remove( keyToMove );
+ while ( !keys.isEmpty()
+ && remainingSnapshotsAndFiles.size() < minCount )
+ {
+ Version keyToMove = keys.last();
- keys.remove( keyToMove );
+ if ( remainingSnapshotsAndFiles.containsKey( keyToMove ) )
+ {
+ remainingSnapshotsAndFiles.get( keyToMove ).addAll(
+ deletableSnapshotsAndFiles.get( keyToMove ) );
}
+ else
+ {
+ remainingSnapshotsAndFiles.put( keyToMove, deletableSnapshotsAndFiles.get( keyToMove ) );
+ }
+
+ deletableSnapshotsAndFiles.remove( keyToMove );
+ keys.remove( keyToMove );
}
}
+ }
+ }
- // NEXUS-814: is this GAV have remaining artifacts?
- boolean gavHasMoreTimestampedSnapshots = remainingSnapshotsAndFiles.size() > 0;
+ private void deleteFiles( final WalkerContext context,
+ final Map<Version, List<StorageFileItem>> deletableVersionedFiles, final int size )
+ {
+ boolean gavHasMoreTimestampedSnapshots = size > 0;
- for ( Version key : deletableSnapshotsAndFiles.keySet() )
- {
+ for ( Version key : deletableVersionedFiles.keySet() )
+ {
- List<StorageFileItem> files = deletableSnapshotsAndFiles.get( key );
- deletedSnapshots++;
+ List<StorageFileItem> files = deletableVersionedFiles.get( key );
+ deletedSnapshots++;
- for ( StorageFileItem file : files )
+ for ( StorageFileItem file : files )
+ {
+ try
{
- try
+ // NEXUS-814: mark that we are deleting a TS snapshot, but there are still remaining
+ // ones in repository.
+ if ( gavHasMoreTimestampedSnapshots )
{
- // NEXUS-814: mark that we are deleting a TS snapshot, but there are still remaining
- // ones in repository.
- if ( gavHasMoreTimestampedSnapshots )
- {
- file.getItemContext().put( MORE_TS_SNAPSHOTS_EXISTS_FOR_GAV, Boolean.TRUE );
- }
-
- gav = (Gav) file.getItemContext().get( Gav.class.getName() );
+ file.getItemContext().put( MORE_TS_SNAPSHOTS_EXISTS_FOR_GAV, Boolean.TRUE );
+ }
- repository.deleteItem( false, createResourceStoreRequest( file, context ) );
+ repository.deleteItem( false, createResourceStoreRequest( file, context ) );
- deletedFiles++;
- }
- catch ( ItemNotFoundException e )
- {
- // NEXUS-5682 Since checksum files are no longer physically represented on the file system,
- // it is expected that they will generate ItemNotFoundException. Log at trace level only for
- // diagnostic purposes.
- if ( getLogger().isTraceEnabled() )
- {
- getLogger().trace( "Could not delete file:", e );
- }
- }
- catch ( Exception e )
+ deletedFiles++;
+ }
+ catch ( ItemNotFoundException e )
+ {
+ // NEXUS-5682 Since checksum files are no longer physically represented on the file system,
+ // it is expected that they will generate ItemNotFoundException. Log at trace level only for
+ // diagnostic purposes.
+ if ( getLogger().isTraceEnabled() )
{
- getLogger().info( "Could not delete file:", e );
+ getLogger().trace( "Could not delete file:", e );
}
}
+ catch ( Exception e )
+ {
+ getLogger().info( "Could not delete file:", e );
+ }
}
}
+ }
- removeDirectoryIfEmpty( repository, coll );
-
- updateMetadataIfNecessary( context, coll );
+ /**
+ * Partition by classifier of StorageFileItems.
+ */
+ private Map<String, Map<Version, List<StorageFileItem>>> partitionFiles(
+ final Map<Version, List<StorageFileItem>> versionedFiles )
+ {
+ Map<String, Map<Version, List<StorageFileItem>>> partitionedFiles = Maps.newHashMap();
+ for ( Map.Entry<Version, List<StorageFileItem>> versionListEntry : versionedFiles.entrySet() )
+ {
+ String classifier = determineClassifier( versionListEntry.getValue() );
+ if ( !partitionedFiles.containsKey( classifier ) )
+ {
+ partitionedFiles.put( classifier, Maps.<Version, List<StorageFileItem>>newHashMap() );
+ }
+ partitionedFiles.get( classifier ).put( versionListEntry.getKey(), versionListEntry.getValue() );
+ }
+ return partitionedFiles;
+ }
+ /**
+ * Find a common classifier for group of files.
+ */
+ private String determineClassifier( final List<StorageFileItem> items )
+ {
+ String classifier = "";
+ for ( StorageFileItem item : items )
+ {
+ Gav gav = (Gav) item.getItemContext().get( Gav.class.getName() );
+ if ( isEmpty( classifier ) && isNotEmpty( gav.getClassifier() ) )
+ {
+ classifier = gav.getClassifier();
+ }
+ else if ( isNotEmpty( classifier ) && isNotEmpty( gav.getClassifier() )
+ && !classifier.equals( gav.getClassifier() ) )
+ {
+ getLogger().warn(
+ "Found more than one classifier in the same snapshot version. Encountered both '{}' and '{}'."
+ + " These will be considered as having no classifier!",
+ classifier, gav.getClassifier() );
+ return "";
+ }
+ }
+ return classifier;
}
private void updateMetadataIfNecessary( WalkerContext context, StorageCollectionItem coll )
@@ -21,6 +21,8 @@
import org.sonatype.nexus.AbstractMavenRepoContentTests;
import org.sonatype.nexus.proxy.ItemNotFoundException;
import org.sonatype.nexus.proxy.ResourceStoreRequest;
+import org.sonatype.nexus.proxy.access.OpenAccessManager;
+import org.sonatype.nexus.proxy.item.StorageItem;
import org.sonatype.nexus.proxy.maven.MavenRepository;
import org.sonatype.nexus.proxy.maven.metadata.operations.MetadataBuilder;
import org.sonatype.nexus.proxy.maven.metadata.operations.MetadataException;
@@ -30,6 +32,7 @@
import java.io.FileInputStream;
import java.io.IOException;
import java.net.URL;
+import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
@@ -741,6 +744,36 @@ public void testEndWithSNAPSHOT()
validateResults( snapshots, expecting );
}
+ /**
+ * This test case uses 3 alpha snapshots and 2 beta snapshots, and after running the task only one of each
+ * artifacts should still be present.
+ * @see <a href='https://issues.sonatype.org/browse/NEXUS-3148'>NEXUS-5094</a>
+ */
+ @Test
+ public void testSnapshotsWithClassifiers()
+ throws Exception
+ {
+ fillInRepo();
+
+ SnapshotRemovalRequest request = new SnapshotRemovalRequest( snapshots.getId(), 1, -1, false );
+ SnapshotRemovalResult snapshotRemovalResult = defaultNexus.removeSnapshots( request );
+ assertThat( snapshotRemovalResult.isSuccessful(), is( true ) );
+
+ HashMap<String, Boolean> expecting = new HashMap<String, Boolean>();
+ expecting.put( "/org/sonatype/test/1.0-SNAPSHOT/test-1.0-20130501.154949-1-alpha.jar", false );
+ expecting.put( "/org/sonatype/test/1.0-SNAPSHOT/test-1.0-20130501.154949-1.pom", false );
+ expecting.put( "/org/sonatype/test/1.0-SNAPSHOT/test-1.0-20130501.154953-2-alpha.jar", false );
+ expecting.put( "/org/sonatype/test/1.0-SNAPSHOT/test-1.0-20130501.154953-2.pom", false );
+ expecting.put( "/org/sonatype/test/1.0-SNAPSHOT/test-1.0-20130501.154956-3-alpha.jar", true );
+ expecting.put( "/org/sonatype/test/1.0-SNAPSHOT/test-1.0-20130501.154956-3.pom", true );
+ expecting.put( "/org/sonatype/test/1.0-SNAPSHOT/test-1.0-20130501.155001-4-beta.jar", false );
+ expecting.put( "/org/sonatype/test/1.0-SNAPSHOT/test-1.0-20130501.155001-4.pom", false );
+ expecting.put( "/org/sonatype/test/1.0-SNAPSHOT/test-1.0-20130501.155005-5-beta.jar", true );
+ expecting.put( "/org/sonatype/test/1.0-SNAPSHOT/test-1.0-20130501.155005-5.pom", true );
+
+ validateResults( snapshots, expecting );
+ }
+
private Metadata readMavenMetadata( File mdFle )
throws MetadataException, IOException
{
@@ -0,0 +1,32 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<metadata modelVersion="1.1.0">
+ <groupId>org.sonatype</groupId>
+ <artifactId>test</artifactId>
+ <version>1.0-SNAPSHOT</version>
+ <versioning>
+ <snapshot>
+ <timestamp>20130501.155005</timestamp>
+ <buildNumber>5</buildNumber>
+ </snapshot>
+ <lastUpdated>20130501155005</lastUpdated>
+ <snapshotVersions>
+ <snapshotVersion>
+ <extension>pom</extension>
+ <value>1.0-20130501.155005-5</value>
+ <updated>20130501155005</updated>
+ </snapshotVersion>
+ <snapshotVersion>
+ <classifier>alpha</classifier>
+ <extension>jar</extension>
+ <value>1.0-20130501.154956-3</value>
+ <updated>20130501154956</updated>
+ </snapshotVersion>
+ <snapshotVersion>
+ <classifier>beta</classifier>
+ <extension>jar</extension>
+ <value>1.0-20130501.155005-5</value>
+ <updated>20130501155005</updated>
+ </snapshotVersion>
+ </snapshotVersions>
+ </versioning>
+</metadata>
@@ -0,0 +1 @@
+99d9bb8d5e12b1bf824604066f439bc1
@@ -0,0 +1 @@
+5bb5a4024fcd907d99da73c263a54ab02dd62d96
@@ -0,0 +1 @@
+f5c4ef7c78a8bd6c136ec3e6ab45cbfc
@@ -0,0 +1 @@
+e68f95d461e6b9228bd73f8c7d5452f12e6dbe68
Oops, something went wrong.

0 comments on commit c5cbb6c

Please sign in to comment.