Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Cleanup: removed separate no-scrape marker file

Changed prefix file format to allow encoding of no-scrape request
using "@ unsupported" directive. Single automatic routing medata
file makes proper synchronization easier to implement and simplifies
overall implementation.

This also eliminated two HEAD requests per remote (due to a bug
original implementation performed HEAD no-scrape file for each
discovery strategy, not once per remote).

Signed-off-by: Igor Fedorenko <igor@ifedorenko.com>
  • Loading branch information...
commit 34c57970caeeb49358c666e08e82140a7fad4e2a 1 parent 954df0f
@ifedorenko ifedorenko authored
Showing with 314 additions and 261 deletions.
  1. +7 −0 nexus-client-core/src/main/java/org/sonatype/nexus/client/core/subsystem/content/Content.java
  2. +24 −4 nexus-client-core/src/main/java/org/sonatype/nexus/client/internal/rest/jersey/subsystem/JerseyContent.java
  3. +1 −16 nexus-core/src/main/java/org/sonatype/nexus/proxy/maven/routing/Config.java
  4. +2 −1  nexus-core/src/main/java/org/sonatype/nexus/proxy/maven/routing/Manager.java
  5. +6 −0 nexus-core/src/main/java/org/sonatype/nexus/proxy/maven/routing/PrefixSource.java
  6. +16 −1 nexus-core/src/main/java/org/sonatype/nexus/proxy/maven/routing/discovery/StrategyResult.java
  7. +6 −0 nexus-core/src/main/java/org/sonatype/nexus/proxy/maven/routing/internal/ArrayListPrefixSource.java
  8. +0 −14 nexus-core/src/main/java/org/sonatype/nexus/proxy/maven/routing/internal/ConfigImpl.java
  9. +37 −2 nexus-core/src/main/java/org/sonatype/nexus/proxy/maven/routing/internal/FilePrefixSource.java
  10. +13 −80 nexus-core/src/main/java/org/sonatype/nexus/proxy/maven/routing/internal/ManagerImpl.java
  11. +1 −1  nexus-core/src/main/java/org/sonatype/nexus/proxy/maven/routing/internal/ProxyRequestFilterImpl.java
  12. +11 −2 nexus-core/src/main/java/org/sonatype/nexus/proxy/maven/routing/internal/RemoteContentDiscovererImpl.java
  13. +14 −30 nexus-core/src/main/java/org/sonatype/nexus/proxy/maven/routing/internal/RemotePrefixFileStrategy.java
  14. +1 −40 nexus-core/src/main/java/org/sonatype/nexus/proxy/maven/routing/internal/RemoteScrapeStrategy.java
  15. +63 −3 nexus-core/src/main/java/org/sonatype/nexus/proxy/maven/routing/internal/TextFilePrefixSourceMarshaller.java
  16. +12 −12 ...e/src/test/java/org/sonatype/nexus/proxy/maven/routing/internal/PrefixFileUpdatePropagationContentTest.java
  17. +6 −6 nexus-core/src/test/java/org/sonatype/nexus/proxy/maven/routing/internal/PrefixFileUpdatePropagationTest.java
  18. +12 −12 nexus-core/src/test/java/org/sonatype/nexus/proxy/maven/routing/internal/RemoteContentDiscovererImplTest.java
  19. +2 −2 nexus-core/src/test/java/org/sonatype/nexus/proxy/maven/routing/internal/RemotePrefixFileStrategyTest.java
  20. +4 −4 ...-core/src/test/java/org/sonatype/nexus/proxy/maven/routing/internal/TextFilePrefixSourceMarshallerTest.java
  21. +6 −0 nexus-core/src/test/java/org/sonatype/nexus/proxy/maven/routing/internal/WritablePrefixSourceModifierTest.java
  22. +25 −0 nexus-test/nexus-core-testsuite/src/test/java/core/routing/ITSupport.java
  23. +14 −10 nexus-test/nexus-core-testsuite/src/test/java/core/routing/WithGroupRepositoryIT.java
  24. +14 −10 nexus-test/nexus-core-testsuite/src/test/java/core/routing/WithHostedRepositoryIT.java
  25. +17 −11 nexus-test/nexus-core-testsuite/src/test/java/core/routing/WithProxyRepositoryIT.java
View
7 nexus-client-core/src/main/java/org/sonatype/nexus/client/core/subsystem/content/Content.java
@@ -14,6 +14,7 @@
import java.io.File;
import java.io.IOException;
+import java.io.OutputStream;
/**
* @since 2.1
@@ -37,6 +38,12 @@ void download( Location location, File target )
void downloadWith( Location location, Directive directive, File target )
throws IOException;
+ /**
+ * @since 2.4
+ */
+ void downloadWith( Location location, Directive directive, OutputStream target )
+ throws IOException;
+
void upload( Location location, File target )
throws IOException;
View
28 nexus-client-core/src/main/java/org/sonatype/nexus/client/internal/rest/jersey/subsystem/JerseyContent.java
@@ -17,6 +17,7 @@
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
+import java.io.OutputStream;
import javax.ws.rs.core.Response;
@@ -131,6 +132,13 @@ public void downloadWith( final Location location, final Directive directive, fi
download( location, toUri( location, directive ), target );
}
+ @Override
+ public void downloadWith( Location location, Directive directive, OutputStream target )
+ throws IOException
+ {
+ download( location, toUri( location, directive ), target );
+ }
+
protected void download( final Location location, final String uri, final File target )
throws IOException
{
@@ -146,6 +154,21 @@ protected void download( final Location location, final String uri, final File t
target.getAbsolutePath() );
}
+ FileOutputStream fos = null;
+ try
+ {
+ fos = new FileOutputStream( target );
+ download( location, uri, fos );
+ }
+ finally
+ {
+ IOUtil.close( fos );
+ }
+ }
+
+ protected void download( final Location location, final String uri, final OutputStream target )
+ throws IOException
+ {
try
{
final ClientResponse response = getNexusClient().uri( uri ).get( ClientResponse.class );
@@ -166,16 +189,13 @@ public String getMessage( final int status )
} );
}
- FileOutputStream fos = null;
try
{
- fos = new FileOutputStream( target );
- IOUtil.copy( response.getEntityInputStream(), fos );
+ IOUtil.copy( response.getEntityInputStream(), target );
}
finally
{
response.close();
- IOUtil.close( fos );
}
}
catch ( ClientHandlerException e )
View
17 nexus-core/src/main/java/org/sonatype/nexus/proxy/maven/routing/Config.java
@@ -12,9 +12,8 @@
*/
package org.sonatype.nexus.proxy.maven.routing;
-
/**
- * WL Configuration.
+ * Automatic routing configuration.
*
* @author cstamas
* @since 2.4
@@ -32,13 +31,6 @@
boolean isFeatureActive();
/**
- * Returns the "no scrape" file path used for flagging a repository to not be scraped.
- *
- * @return the no scrape file path.
- */
- String getLocalNoScrapeFlagPath();
-
- /**
* Returns the local file path to publish prefix file.
*
* @return the prefix file path.
@@ -46,13 +38,6 @@
String getLocalPrefixFilePath();
/**
- * Returns the path that should be checked for published no scrape flag on remote.
- *
- * @return the array of paths to have checked on remote.
- */
- String getRemoteNoScrapeFlagPath();
-
- /**
* Returns the path that should be checked for published prefix file on remote.
*
* @return the array of paths to have checked on remote.
View
3  nexus-core/src/main/java/org/sonatype/nexus/proxy/maven/routing/Manager.java
@@ -176,7 +176,8 @@ boolean revokeEntry( final MavenHostedRepository mavenHostedRepository, String e
/**
* Returns {@link PrefixSource} for given {@link MavenRepository}.For the existence of the WL in question (if you
- * want to read it), check {@link PrefixSource#exists()} method! Never returns {@code null}.
+ * want to read it), check {@link PrefixSource#exists()} and {@link PrefixSource#supported()} method! Never returns
+ * {@code null}.
*
* @param mavenRepository
* @return the {@link PrefixSource} for given repository.
View
6 nexus-core/src/main/java/org/sonatype/nexus/proxy/maven/routing/PrefixSource.java
@@ -32,6 +32,12 @@
boolean exists();
/**
+ * Returns {@code true} if automatic routing is supported. Automatic routing is considered unsupported if
+ * {@link #exists()} returns {@code false}.
+ */
+ boolean supported();
+
+ /**
* Reads entries for this source, of {@code null} if not exists ({@link #exists()} returns {@code false} in this
* case).
*
View
17 nexus-core/src/main/java/org/sonatype/nexus/proxy/maven/routing/discovery/StrategyResult.java
@@ -27,16 +27,19 @@
private final PrefixSource prefixSource;
+ private final boolean routingEnabled;
+
/**
* Constructor.
*
* @param message
* @param prefixSource
*/
- public StrategyResult( final String message, final PrefixSource prefixSource )
+ public StrategyResult( final String message, final PrefixSource prefixSource, boolean routingEnabled )
{
this.message = checkNotNull( message );
this.prefixSource = checkNotNull( prefixSource );
+ this.routingEnabled = routingEnabled;
}
/**
@@ -58,4 +61,16 @@ public PrefixSource getPrefixSource()
{
return prefixSource;
}
+
+ /**
+ * Returns <code>false</code> if remote explicitly requested automatic routing to be disabled. This normally
+ * indicates that remote is not able to provide reliable path prefix information. For example, remote itself is a
+ * proxy of a repository that does not provide prefix file and cannot be scraped.
+ *
+ * @return
+ */
+ public boolean isRoutingEnabled()
+ {
+ return routingEnabled;
+ }
}
View
6 nexus-core/src/main/java/org/sonatype/nexus/proxy/maven/routing/internal/ArrayListPrefixSource.java
@@ -63,6 +63,12 @@ public boolean exists()
}
@Override
+ public boolean supported()
+ {
+ return true;
+ }
+
+ @Override
public List<String> readEntries()
throws IOException
{
View
14 nexus-core/src/main/java/org/sonatype/nexus/proxy/maven/routing/internal/ConfigImpl.java
@@ -49,8 +49,6 @@
*/
public static final String FEATURE_ACTIVE_KEY = Config.class.getName() + ".featureActive";
- private static final String LOCAL_NO_SCRAPE_FLAG_PATH = "/.meta/noscrape.txt";
-
private static final String LOCAL_PREFIX_FILE_PATH = "/.meta/prefixes.txt";
private static final int REMOTE_SCRAPE_DEPTH = SystemPropertiesHelper.getInteger( Config.class.getName()
@@ -95,24 +93,12 @@ public boolean isFeatureActive()
}
@Override
- public String getLocalNoScrapeFlagPath()
- {
- return LOCAL_NO_SCRAPE_FLAG_PATH;
- }
-
- @Override
public String getLocalPrefixFilePath()
{
return LOCAL_PREFIX_FILE_PATH;
}
@Override
- public String getRemoteNoScrapeFlagPath()
- {
- return LOCAL_NO_SCRAPE_FLAG_PATH;
- }
-
- @Override
public String getRemotePrefixFilePath()
{
return LOCAL_PREFIX_FILE_PATH;
View
39 nexus-core/src/main/java/org/sonatype/nexus/proxy/maven/routing/internal/FilePrefixSource.java
@@ -32,9 +32,9 @@
import org.sonatype.nexus.proxy.item.StorageFileItem;
import org.sonatype.nexus.proxy.item.StorageItem;
import org.sonatype.nexus.proxy.maven.MavenRepository;
-import org.sonatype.nexus.proxy.maven.routing.PrefixSource;
import org.sonatype.nexus.proxy.maven.routing.Config;
import org.sonatype.nexus.proxy.maven.routing.Manager;
+import org.sonatype.nexus.proxy.maven.routing.PrefixSource;
import org.sonatype.nexus.proxy.maven.routing.WritablePrefixSource;
import org.sonatype.nexus.proxy.storage.UnsupportedStorageOperationException;
@@ -133,6 +133,33 @@ public Boolean call()
}
@Override
+ public boolean supported()
+ {
+ try
+ {
+ return doReadProtected( new Callable<Boolean>()
+ {
+ @Override
+ public Boolean call()
+ throws IOException
+ {
+ StorageFileItem file = getFileItem();
+ if ( file != null )
+ {
+ return getPrefixSourceMarshaller().read( file ).supported();
+ }
+ return false;
+ }
+ } );
+ }
+ catch ( IOException e )
+ {
+ // bam
+ }
+ return false;
+ }
+
+ @Override
public long getLostModifiedTimestamp()
{
try
@@ -177,7 +204,7 @@ public Long call()
{
return null;
}
- return getPrefixSourceMarshaller().read( file );
+ return getPrefixSourceMarshaller().read( file ).entries();
}
} );
}
@@ -318,4 +345,12 @@ protected void deleteFileItem()
// ignore
}
}
+
+ public void writeUnsupported()
+ throws IOException
+ {
+ final ByteArrayOutputStream bos = new ByteArrayOutputStream();
+ getPrefixSourceMarshaller().writeUnsupported( bos );
+ putFileItem( new PreparedContentLocator( new ByteArrayInputStream( bos.toByteArray() ), "text/plain" ) );
+ }
}
View
93 nexus-core/src/main/java/org/sonatype/nexus/proxy/maven/routing/internal/ManagerImpl.java
@@ -31,17 +31,12 @@
import org.sonatype.nexus.ApplicationStatusSource;
import org.sonatype.nexus.configuration.application.ApplicationConfiguration;
import org.sonatype.nexus.logging.AbstractLoggingComponent;
-import org.sonatype.nexus.proxy.IllegalOperationException;
-import org.sonatype.nexus.proxy.ItemNotFoundException;
-import org.sonatype.nexus.proxy.ResourceStoreRequest;
import org.sonatype.nexus.proxy.access.Action;
import org.sonatype.nexus.proxy.events.NexusStartedEvent;
import org.sonatype.nexus.proxy.events.NexusStoppedEvent;
import org.sonatype.nexus.proxy.events.RepositoryItemEvent;
-import org.sonatype.nexus.proxy.item.DefaultStorageFileItem;
import org.sonatype.nexus.proxy.item.RepositoryItemUidLock;
import org.sonatype.nexus.proxy.item.StorageFileItem;
-import org.sonatype.nexus.proxy.item.StringContentLocator;
import org.sonatype.nexus.proxy.maven.AbstractMavenRepositoryConfiguration;
import org.sonatype.nexus.proxy.maven.MavenGroupRepository;
import org.sonatype.nexus.proxy.maven.MavenHostedRepository;
@@ -49,20 +44,20 @@
import org.sonatype.nexus.proxy.maven.MavenRepository;
import org.sonatype.nexus.proxy.maven.MavenShadowRepository;
import org.sonatype.nexus.proxy.maven.maven2.Maven2ContentClass;
-import org.sonatype.nexus.proxy.maven.routing.PrefixSource;
import org.sonatype.nexus.proxy.maven.routing.Config;
import org.sonatype.nexus.proxy.maven.routing.DiscoveryConfig;
import org.sonatype.nexus.proxy.maven.routing.DiscoveryStatus;
+import org.sonatype.nexus.proxy.maven.routing.DiscoveryStatus.DStatus;
import org.sonatype.nexus.proxy.maven.routing.Manager;
+import org.sonatype.nexus.proxy.maven.routing.PrefixSource;
import org.sonatype.nexus.proxy.maven.routing.PublishingStatus;
-import org.sonatype.nexus.proxy.maven.routing.RoutingStatus;
-import org.sonatype.nexus.proxy.maven.routing.DiscoveryStatus.DStatus;
import org.sonatype.nexus.proxy.maven.routing.PublishingStatus.PStatus;
+import org.sonatype.nexus.proxy.maven.routing.RoutingStatus;
import org.sonatype.nexus.proxy.maven.routing.discovery.DiscoveryResult;
+import org.sonatype.nexus.proxy.maven.routing.discovery.DiscoveryResult.Outcome;
import org.sonatype.nexus.proxy.maven.routing.discovery.LocalContentDiscoverer;
import org.sonatype.nexus.proxy.maven.routing.discovery.RemoteContentDiscoverer;
import org.sonatype.nexus.proxy.maven.routing.discovery.RemoteStrategy;
-import org.sonatype.nexus.proxy.maven.routing.discovery.DiscoveryResult.Outcome;
import org.sonatype.nexus.proxy.maven.routing.events.PrefixFilePublishedRepositoryEvent;
import org.sonatype.nexus.proxy.maven.routing.events.PrefixFileUnpublishedRepositoryEvent;
import org.sonatype.nexus.proxy.registry.RepositoryRegistry;
@@ -71,7 +66,6 @@
import org.sonatype.nexus.proxy.repository.ProxyMode;
import org.sonatype.nexus.proxy.repository.Repository;
import org.sonatype.nexus.proxy.repository.ShadowRepository;
-import org.sonatype.nexus.proxy.storage.UnsupportedStorageOperationException;
import org.sonatype.nexus.proxy.utils.RepositoryStringUtils;
import org.sonatype.nexus.threads.NexusThreadFactory;
import org.sonatype.nexus.util.task.LoggingProgressListener;
@@ -350,7 +344,7 @@ protected boolean doInitializePrefixFileOnStartup( final MavenRepository mavenRe
final PrefixSource prefixSource = getPrefixSourceFor( mavenRepository );
try
{
- if ( prefixSource.exists() )
+ if ( prefixSource.supported() )
{
// good, we assume is up to date, which should be unless user tampered with it
// in that case, just delete it + update and should be fixed.
@@ -501,7 +495,7 @@ public void forceProxyQuickUpdatePrefixFile( final MavenProxyRepository mavenPro
constrainedExecutor.cancelRunningWithKey( mavenProxyRepository.getId() );
final PrefixSource prefixSource =
updateProxyPrefixFile( mavenProxyRepository, Collections.singletonList( quickRemoteStrategy ) );
- if ( prefixSource != null )
+ if ( prefixSource != null && prefixSource.supported() )
{
getLogger().info( "Updated and published prefix file of {}",
RepositoryStringUtils.getHumanizedNameString( mavenProxyRepository ) );
@@ -649,7 +643,7 @@ else if ( mavenRepository.getRepositoryKind().isFacetAvailable( MavenHostedRepos
RepositoryStringUtils.getFullHumanizedNameString( mavenRepository ) );
return;
}
- if ( prefixSource != null )
+ if ( prefixSource != null && prefixSource.supported() )
{
if ( notify )
{
@@ -790,7 +784,7 @@ protected PrefixSource updateGroupPrefixFile( final MavenGroupRepository mavenGr
lock.lock( Action.read );
try
{
- if ( !memberEntrySource.exists() )
+ if ( !memberEntrySource.supported() )
{
allMembersHavePublished = false;
break;
@@ -833,7 +827,7 @@ public RoutingStatus getStatusFor( final MavenRepository mavenRepository )
// publish status
final FilePrefixSource publishedEntrySource = getPrefixSourceFor( mavenRepository );
- if ( !publishedEntrySource.exists() )
+ if ( !publishedEntrySource.supported() )
{
final String message;
if ( isMavenRepositorySupported( mavenRepository ) )
@@ -849,7 +843,7 @@ public RoutingStatus getStatusFor( final MavenRepository mavenRepository )
if ( null != memberMavenRepository )
{
final PrefixSource ps = getPrefixSourceFor( memberMavenRepository );
- if ( !ps.exists() )
+ if ( !ps.supported() )
{
membersWithoutPrefixFiles.add( memberMavenRepository.getName() );
}
@@ -1002,7 +996,7 @@ public boolean offerEntry( final MavenHostedRepository mavenHostedRepository, fi
lock.lock( Action.read );
try
{
- if ( !prefixSource.exists() )
+ if ( !prefixSource.supported() )
{
return false;
}
@@ -1051,7 +1045,7 @@ public boolean revokeEntry( final MavenHostedRepository mavenHostedRepository, f
lock.lock( Action.read );
try
{
- if ( !prefixSource.exists() )
+ if ( !prefixSource.supported() )
{
return false;
}
@@ -1111,9 +1105,6 @@ protected void publish( final MavenRepository mavenRepository, final PrefixSourc
throw e;
}
- // unset noscrape flag
- removeNoscrapeFlag( mavenRepository );
-
// event
eventBus.post( new PrefixFilePublishedRepositoryEvent( mavenRepository, prefixesFile ) );
@@ -1134,15 +1125,7 @@ public void unpublish( final MavenRepository mavenRepository )
protected void unpublish( final MavenRepository mavenRepository, final boolean propagate )
throws IOException
{
- // delete (if any) published files, even those that user might manually put there
- getPrefixSourceFor( mavenRepository ).delete();
-
- // TODO: We do this due to RemotePrefixFileStrategy, but this is now scattered (that one may write these file,
- // and here we are cleaning them)
- new FilePrefixSource( mavenRepository, config.getRemotePrefixFilePath(), config ).delete();
-
- // set noscrape flag
- addNoscrapeFlag( mavenRepository );
+ getPrefixSourceFor( mavenRepository ).writeUnsupported();
// event
eventBus.post( new PrefixFileUnpublishedRepositoryEvent( mavenRepository ) );
@@ -1173,56 +1156,6 @@ protected void propagatePrefixFileUpdateOf( final MavenRepository mavenRepositor
// ==
- protected void addNoscrapeFlag( final MavenRepository mavenRepository )
- throws IOException
- {
- final ResourceStoreRequest request = new ResourceStoreRequest( config.getLocalNoScrapeFlagPath() );
- request.setRequestLocalOnly( true );
- request.setRequestGroupLocalOnly( true );
- final DefaultStorageFileItem file =
- new DefaultStorageFileItem( mavenRepository, new ResourceStoreRequest( config.getLocalNoScrapeFlagPath() ),
- true, true, new StringContentLocator( "noscrape" ) );
- try
- {
- mavenRepository.storeItem( true, file );
- }
- catch ( UnsupportedStorageOperationException e )
- {
- // eh?
- }
- catch ( IllegalOperationException e )
- {
- // eh?
- }
- }
-
- @SuppressWarnings( "deprecation" )
- protected void removeNoscrapeFlag( final MavenRepository mavenRepository )
- throws IOException
- {
- final ResourceStoreRequest request = new ResourceStoreRequest( config.getLocalNoScrapeFlagPath() );
- request.setRequestLocalOnly( true );
- request.setRequestGroupLocalOnly( true );
- try
- {
- mavenRepository.deleteItem( true, request );
- }
- catch ( ItemNotFoundException e )
- {
- // ignore
- }
- catch ( UnsupportedStorageOperationException e )
- {
- // eh?
- }
- catch ( IllegalOperationException e )
- {
- // eh?
- }
- }
-
- // ==
-
@Override
public boolean isEventAboutPrefixFile( RepositoryItemEvent evt )
{
View
2  nexus-core/src/main/java/org/sonatype/nexus/proxy/maven/routing/internal/ProxyRequestFilterImpl.java
@@ -108,7 +108,7 @@ protected void buildPathMatcherFor( final MavenProxyRepository mavenProxyReposit
try
{
final PrefixSource prefixSource = manager.getPrefixSourceFor( mavenProxyRepository );
- if ( prefixSource.exists() )
+ if ( prefixSource.supported() )
{
final PathMatcher pathMatcher = new PathMatcher( prefixSource.readEntries(), Integer.MAX_VALUE );
pathMatchers.put( mavenProxyRepository.getId(), pathMatcher );
View
13 nexus-core/src/main/java/org/sonatype/nexus/proxy/maven/routing/internal/RemoteContentDiscovererImpl.java
@@ -76,8 +76,17 @@ public RemoteContentDiscovererImpl( final List<RemoteStrategy> remoteStrategies
try
{
final StrategyResult strategyResult = strategy.discover( mavenProxyRepository );
- discoveryResult.recordSuccess( strategy.getId(), strategyResult.getMessage(),
- strategyResult.getPrefixSource() );
+ if ( strategyResult.isRoutingEnabled() )
+ {
+ discoveryResult.recordSuccess( strategy.getId(), strategyResult.getMessage(),
+ strategyResult.getPrefixSource() );
+ }
+ else
+ {
+ // the strategy explicitly requested to disable automatic routing for the repository
+ discoveryResult.recordFailure( strategy.getId(), strategyResult.getMessage() );
+ break;
+ }
}
catch ( StrategyFailedException e )
{
View
44 nexus-core/src/main/java/org/sonatype/nexus/proxy/maven/routing/internal/RemotePrefixFileStrategy.java
@@ -34,6 +34,7 @@
import org.sonatype.nexus.proxy.maven.routing.discovery.RemoteStrategy;
import org.sonatype.nexus.proxy.maven.routing.discovery.StrategyFailedException;
import org.sonatype.nexus.proxy.maven.routing.discovery.StrategyResult;
+import org.sonatype.nexus.proxy.maven.routing.internal.TextFilePrefixSourceMarshaller.Result;
/**
* Remote prefix file strategy.
@@ -82,23 +83,22 @@ public StrategyResult discover( final MavenProxyRepository mavenProxyRepository
{
getLogger().debug( "Remote prefix on {} at path {} found!", mavenProxyRepository, path );
long prefixFileAgeInDays = ( System.currentTimeMillis() - item.getModified() ) / 86400000L;
- final PrefixSource prefixSource = createPrefixSource( mavenProxyRepository, path );
- if ( prefixSource != null )
+ Result unmarshalled = new TextFilePrefixSourceMarshaller( config ).read( item );
+ if ( !unmarshalled.supported() )
{
- if ( prefixFileAgeInDays < 1 )
- {
- return new StrategyResult(
- "Remote publishes prefix file (is less than a day old), using it.", prefixSource );
- }
- else
- {
- return new StrategyResult( "Remote publishes prefix file (is " + prefixFileAgeInDays
- + " days old), using it.", prefixSource );
- }
+ return new StrategyResult( "Remote disabled automatic routing", null, false );
+ }
+
+ final PrefixSource prefixSource = new ArrayListPrefixSource( unmarshalled.entries() );
+ if ( prefixFileAgeInDays < 1 )
+ {
+ return new StrategyResult( "Remote publishes prefix file (is less than a day old), using it.",
+ prefixSource, true );
}
else
{
- getLogger().info( "Prefix file retrieved from {} is corrupt, skipping it.", item.getRemoteUrl() );
+ return new StrategyResult( "Remote publishes prefix file (is " + prefixFileAgeInDays
+ + " days old), using it.", prefixSource, true );
}
}
}
@@ -106,27 +106,11 @@ public StrategyResult discover( final MavenProxyRepository mavenProxyRepository
{
uid.getLock().unlock();
}
- throw new StrategyFailedException( "Remote does not publish prefix files on paths " + path );
+ throw new StrategyFailedException( "Remote does not publish prefix files on path " + path );
}
// ==
- protected FilePrefixSource createPrefixSource( final MavenProxyRepository mavenProxyRepository, final String path )
- throws IOException
- {
- final FilePrefixSource result = new FilePrefixSource( mavenProxyRepository, path, config );
- try
- {
- result.readEntries();
- }
- catch ( InvalidInputException e )
- {
- result.delete();
- throw e;
- }
- return result;
- }
-
protected StorageFileItem retrieveFromRemoteIfExists( final MavenProxyRepository mavenProxyRepository,
final String path )
throws IOException
View
41 nexus-core/src/main/java/org/sonatype/nexus/proxy/maven/routing/internal/RemoteScrapeStrategy.java
@@ -25,10 +25,7 @@
import javax.inject.Named;
import javax.inject.Singleton;
-import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
-import org.apache.http.client.methods.HttpHead;
-import org.apache.http.util.EntityUtils;
import org.sonatype.nexus.proxy.maven.MavenProxyRepository;
import org.sonatype.nexus.proxy.maven.routing.Config;
import org.sonatype.nexus.proxy.maven.routing.discovery.RemoteStrategy;
@@ -106,11 +103,6 @@ public StrategyResult discover( final MavenProxyRepository mavenProxyRepository
httpClientManager.create( mavenProxyRepository, mavenProxyRepository.getRemoteStorageContext() );
final ScrapeContext context =
new ScrapeContext( mavenProxyRepository, httpClient, config.getRemoteScrapeDepth() );
- if ( isMarkedForNoScrape( context ) )
- {
- getLogger().debug( "Remote {} marked as no-scrape, giving up.", mavenProxyRepository );
- throw new StrategyFailedException( "Remote forbids scraping, is flagged as \"no-scrape\"." );
- }
final Page rootPage = Page.getPageFor( context, remoteRepositoryRootUrl );
final ArrayList<Scraper> appliedScrapers = new ArrayList<Scraper>( scrapers );
Collections.sort( appliedScrapers, new PriorityOrderingComparator<Scraper>() );
@@ -124,7 +116,7 @@ public StrategyResult discover( final MavenProxyRepository mavenProxyRepository
{
getLogger().debug( "Remote scraping {} with Scraper {} succeeded.", mavenProxyRepository,
scraper.getId() );
- return new StrategyResult( context.getMessage(), context.getPrefixSource() );
+ return new StrategyResult( context.getMessage(), context.getPrefixSource(), true );
}
else
{
@@ -139,35 +131,4 @@ public StrategyResult discover( final MavenProxyRepository mavenProxyRepository
getLogger().debug( "Not possible remote scrape of {}, no scraper succeeded.", mavenProxyRepository );
throw new StrategyFailedException( "No scraper was able to scrape remote (or remote prevents scraping)." );
}
-
- // ==
-
- protected boolean isMarkedForNoScrape( final ScrapeContext context )
- throws IOException
- {
- String noscrapeFlag = config.getRemoteNoScrapeFlagPath();
- while ( noscrapeFlag.startsWith( "/" ) )
- {
- noscrapeFlag = noscrapeFlag.substring( 1 );
- }
- final String flagRemoteUrl = context.getRemoteRepositoryRootUrl() + noscrapeFlag;
- HttpResponse response = null;
- try
- {
- final HttpHead head = new HttpHead( flagRemoteUrl );
- response = context.executeHttpRequest( head );
- if ( response.getStatusLine().getStatusCode() > 199 && response.getStatusLine().getStatusCode() < 300 )
- {
- return true;
- }
- }
- finally
- {
- if ( response != null )
- {
- EntityUtils.consumeQuietly( response.getEntity() );
- }
- }
- return false;
- }
}
View
66 ...s-core/src/main/java/org/sonatype/nexus/proxy/maven/routing/internal/TextFilePrefixSourceMarshaller.java
@@ -17,6 +17,7 @@
import static org.sonatype.nexus.util.PathUtils.pathFrom;
import java.io.BufferedReader;
+import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStream;
@@ -24,6 +25,7 @@
import java.io.PrintWriter;
import java.nio.charset.Charset;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
import org.sonatype.nexus.proxy.item.StorageFileItem;
@@ -46,12 +48,40 @@
protected static final Charset CHARSET = Charset.forName( "UTF-8" );
+ /**
+ * Directive that indicates that automatic routing is not supported for the repository. Must be the first line of
+ * the prefix file.
+ */
+ protected static final String UNSUPPORTED = "@ unsupported";
+
private final int prefixFileMaxSize;
private final int prefixFileMaxLineLength;
private final int prefixFileMaxEntryCount;
+ public interface Result
+ {
+ boolean supported();
+
+ List<String> entries();
+ }
+
+ private static final Result RESULT_UNSUPPORTED = new Result()
+ {
+ @Override
+ public boolean supported()
+ {
+ return false;
+ }
+
+ @Override
+ public List<String> entries()
+ {
+ return Collections.emptyList();
+ }
+ };
+
/**
* Constructor.
*
@@ -97,7 +127,7 @@ public void write( final List<String> entries, final OutputStream outputStream )
* @throws InvalidInputException
* @throws IOException
*/
- public final List<String> read( final StorageFileItem file )
+ public final Result read( final StorageFileItem file )
throws InvalidInputException, IOException
{
if ( file.getLength() > prefixFileMaxSize )
@@ -111,6 +141,12 @@ public void write( final List<String> entries, final OutputStream outputStream )
final ArrayList<String> entries = new ArrayList<String>();
reader = new BufferedReader( new InputStreamReader( file.getInputStream(), CHARSET ) );
String line = reader.readLine();
+
+ if ( UNSUPPORTED.equals( line ) )
+ {
+ return RESULT_UNSUPPORTED;
+ }
+
while ( line != null )
{
// trim
@@ -127,7 +163,7 @@ public void write( final List<String> entries, final OutputStream outputStream )
if ( !CharMatcher.ASCII.matchesAllOf( line ) )
{
throw new InvalidInputException(
- "Prefix file contains non-ASCII characters, refusing to load the file." );
+ "Prefix file contains non-ASCII characters, refusing to load the file." );
}
// Igor's find command makes path like "./org/apache/"
while ( line.startsWith( "." ) )
@@ -148,11 +184,35 @@ public void write( final List<String> entries, final OutputStream outputStream )
+ prefixFileMaxEntryCount + "), refusing to load it." );
}
}
- return entries;
+ return new Result()
+ {
+ @Override
+ public boolean supported()
+ {
+ return true;
+ }
+
+ @Override
+ public List<String> entries()
+ {
+ return entries;
+ }
+ };
}
finally
{
Closeables.closeQuietly( reader );
}
}
+
+ public void writeUnsupported( ByteArrayOutputStream outputStream )
+ {
+ final PrintWriter printWriter = new PrintWriter( new OutputStreamWriter( outputStream, CHARSET ) );
+ printWriter.println( UNSUPPORTED );
+ for ( String header : HEADERS )
+ {
+ printWriter.println( header );
+ }
+ printWriter.flush();
+ }
}
View
24 ...rc/test/java/org/sonatype/nexus/proxy/maven/routing/internal/PrefixFileUpdatePropagationContentTest.java
@@ -300,7 +300,7 @@ public void contentUponBoot()
wm.getPrefixSourceFor( getRepositoryRegistry().getRepositoryWithFacet( GROUP1_REPO_ID,
MavenRepository.class ) );
- assertThat( "Group1 should have WL", groupPrefixSource.exists() );
+ assertThat( "Group1 should have WL", groupPrefixSource.supported() );
final List<String> groupEntries = groupPrefixSource.readEntries();
assertThat( groupEntries, hasSize( 3 ) );
@@ -314,7 +314,7 @@ public void contentUponBoot()
wm.getPrefixSourceFor( getRepositoryRegistry().getRepositoryWithFacet( GROUP2_REPO_ID,
MavenRepository.class ) );
- assertThat( "Group2 should have WL", groupPrefixSource.exists() );
+ assertThat( "Group2 should have WL", groupPrefixSource.supported() );
final List<String> groupEntries = groupPrefixSource.readEntries();
assertThat( groupEntries, hasSize( 3 ) );
@@ -342,7 +342,7 @@ public void contentOnMemberRemoval()
wm.getPrefixSourceFor( getRepositoryRegistry().getRepositoryWithFacet( GROUP1_REPO_ID,
MavenRepository.class ) );
- assertThat( "Group1 should have WL", groupPrefixSource.exists() );
+ assertThat( "Group1 should have WL", groupPrefixSource.supported() );
final List<String> groupEntries = groupPrefixSource.readEntries();
assertThat( groupEntries, hasSize( 1 ) );
@@ -354,7 +354,7 @@ public void contentOnMemberRemoval()
wm.getPrefixSourceFor( getRepositoryRegistry().getRepositoryWithFacet( GROUP2_REPO_ID,
MavenRepository.class ) );
- assertThat( "Group2 should have WL", groupPrefixSource.exists() );
+ assertThat( "Group2 should have WL", groupPrefixSource.supported() );
final List<String> groupEntries = groupPrefixSource.readEntries();
assertThat( groupEntries, hasSize( 1 ) );
@@ -381,7 +381,7 @@ public void contentOnMemberAddition()
wm.getPrefixSourceFor( getRepositoryRegistry().getRepositoryWithFacet( GROUP1_REPO_ID,
MavenRepository.class ) );
- assertThat( "Group1 should have WL", groupPrefixSource.exists() );
+ assertThat( "Group1 should have WL", groupPrefixSource.supported() );
final List<String> groupEntries = groupPrefixSource.readEntries();
assertThat( groupEntries, hasSize( 4 ) );
@@ -396,7 +396,7 @@ public void contentOnMemberAddition()
wm.getPrefixSourceFor( getRepositoryRegistry().getRepositoryWithFacet( GROUP2_REPO_ID,
MavenRepository.class ) );
- assertThat( "Group2 should have WL", groupPrefixSource.exists() );
+ assertThat( "Group2 should have WL", groupPrefixSource.supported() );
final List<String> groupEntries = groupPrefixSource.readEntries();
assertThat( groupEntries, hasSize( 4 ) );
@@ -428,7 +428,7 @@ public void contentOnMemberAdditionAndRemoval()
wm.getPrefixSourceFor( getRepositoryRegistry().getRepositoryWithFacet( GROUP1_REPO_ID,
MavenRepository.class ) );
- assertThat( "Group1 should have WL", groupPrefixSource.exists() );
+ assertThat( "Group1 should have WL", groupPrefixSource.supported() );
final List<String> groupEntries = groupPrefixSource.readEntries();
assertThat( groupEntries, hasSize( 2 ) );
@@ -441,7 +441,7 @@ public void contentOnMemberAdditionAndRemoval()
wm.getPrefixSourceFor( getRepositoryRegistry().getRepositoryWithFacet( GROUP2_REPO_ID,
MavenRepository.class ) );
- assertThat( "Group2 should have WL", groupPrefixSource.exists() );
+ assertThat( "Group2 should have WL", groupPrefixSource.supported() );
final List<String> groupEntries = groupPrefixSource.readEntries();
assertThat( groupEntries, hasSize( 2 ) );
@@ -472,7 +472,7 @@ public void contentOnMemberAdditionAndRemovalInDifferentGroups()
wm.getPrefixSourceFor( getRepositoryRegistry().getRepositoryWithFacet( GROUP1_REPO_ID,
MavenRepository.class ) );
- assertThat( "Group1 should have WL", groupPrefixSource.exists() );
+ assertThat( "Group1 should have WL", groupPrefixSource.supported() );
final List<String> groupEntries = groupPrefixSource.readEntries();
assertThat( groupEntries, hasSize( 1 ) );
@@ -484,7 +484,7 @@ public void contentOnMemberAdditionAndRemovalInDifferentGroups()
wm.getPrefixSourceFor( getRepositoryRegistry().getRepositoryWithFacet( GROUP2_REPO_ID,
MavenRepository.class ) );
- assertThat( "Group2 should have WL", groupPrefixSource.exists() );
+ assertThat( "Group2 should have WL", groupPrefixSource.supported() );
final List<String> groupEntries = groupPrefixSource.readEntries();
assertThat( groupEntries, hasSize( 2 ) );
@@ -515,7 +515,7 @@ public void contentOnMemberAdditionOfSameInDifferentGroups()
wm.getPrefixSourceFor( getRepositoryRegistry().getRepositoryWithFacet( GROUP1_REPO_ID,
MavenRepository.class ) );
- assertThat( "Group1 should have WL", groupPrefixSource.exists() );
+ assertThat( "Group1 should have WL", groupPrefixSource.supported() );
final List<String> groupEntries = groupPrefixSource.readEntries();
assertThat( groupEntries, hasSize( 4 ) );
@@ -531,7 +531,7 @@ public void contentOnMemberAdditionOfSameInDifferentGroups()
wm.getPrefixSourceFor( getRepositoryRegistry().getRepositoryWithFacet( GROUP2_REPO_ID,
MavenRepository.class ) );
- assertThat( "Group2 should have WL", groupPrefixSource.exists() );
+ assertThat( "Group2 should have WL", groupPrefixSource.supported() );
final List<String> groupEntries = groupPrefixSource.readEntries();
assertThat( groupEntries, hasSize( 4 ) );
View
12 ...-core/src/test/java/org/sonatype/nexus/proxy/maven/routing/internal/PrefixFileUpdatePropagationTest.java
@@ -224,10 +224,10 @@ public void smoke()
wm.getPrefixSourceFor( getRepositoryRegistry().getRepositoryWithFacet( GROUP_REPO_ID,
MavenRepository.class ) );
- assertThat( "Hosted should have ES", hostedEntrySource.exists() );
- assertThat( "Proxy should not have ES", !proxyEntrySource.exists() ); // as we serve 404s for prefix
+ assertThat( "Hosted should have ES", hostedEntrySource.supported() );
+ assertThat( "Proxy should not have ES", !proxyEntrySource.supported() ); // as we serve noscrape prefix
// file
- assertThat( "Group cannot have ES", !groupEntrySource.exists() ); // as proxy member does not have WL
+ assertThat( "Group cannot have ES", !groupEntrySource.supported() ); // as proxy member does not have WL
}
finally
{
@@ -258,9 +258,9 @@ public void smoke()
wm.getPrefixSourceFor( getRepositoryRegistry().getRepositoryWithFacet( GROUP_REPO_ID,
MavenRepository.class ) );
- assertThat( "Hosted should have ES", hostedEntrySource.exists() );
- assertThat( "Proxy should have ES", proxyEntrySource.exists() ); // as we did serve file ok
- assertThat( "Group should have ES", groupEntrySource.exists() ); // as all member should have it
+ assertThat( "Hosted should have ES", hostedEntrySource.supported() );
+ assertThat( "Proxy should have ES", proxyEntrySource.supported() ); // as we did serve file ok
+ assertThat( "Group should have ES", groupEntrySource.supported() ); // as all member should have it
// GROUP wl must have 4 entries: 1 from hosted (/com/sonatype) + 3 from proxied prefix file
final List<String> groupEntries = groupEntrySource.readEntries();
View
24 ...-core/src/test/java/org/sonatype/nexus/proxy/maven/routing/internal/RemoteContentDiscovererImplTest.java
@@ -239,9 +239,9 @@ public void noPrefixFilesServed()
wm.getPrefixSourceFor( getRepositoryRegistry().getRepositoryWithFacet( GROUP_REPO_ID,
MavenRepository.class ) );
- assertThat( "Proxy1 should not have ES", !proxy1EntrySource.exists() ); // we serve 404
- assertThat( "Proxy2 should not have ES", !proxy2EntrySource.exists() ); // we serve 404
- assertThat( "Group cannot have ES", !groupEntrySource.exists() ); // as proxy member does not have WL
+ assertThat( "Proxy1 should not have ES", !proxy1EntrySource.supported() ); // we served noscrape file
+ assertThat( "Proxy2 should not have ES", !proxy2EntrySource.supported() ); // we served noscrape file
+ assertThat( "Group cannot have ES", !groupEntrySource.supported() ); // as proxy member are disabled
}
}
finally
@@ -278,9 +278,9 @@ public void oneOutOfTwoPrefixFilesServed()
wm.getPrefixSourceFor( getRepositoryRegistry().getRepositoryWithFacet( GROUP_REPO_ID,
MavenRepository.class ) );
- assertThat( "Proxy1 should have ES", proxy1EntrySource.exists() ); // we served prefix file
- assertThat( "Proxy2 should not have ES", !proxy2EntrySource.exists() ); // we serve 404
- assertThat( "Group cannot have ES", !groupEntrySource.exists() ); // as proxy member does not have WL
+ assertThat( "Proxy1 should have ES", proxy1EntrySource.supported() ); // we serve prefix file
+ assertThat( "Proxy2 should not have ES", !proxy2EntrySource.supported() ); // we serve noscrape file
+ assertThat( "Group cannot have ES", !groupEntrySource.supported() ); // we serve noscrape file
}
}
finally
@@ -325,9 +325,9 @@ public void twoOutOfTwoPrefixFilesServed()
wm.getPrefixSourceFor( getRepositoryRegistry().getRepositoryWithFacet( GROUP_REPO_ID,
MavenRepository.class ) );
- assertThat( "Proxy1 should have ES", proxy1EntrySource.exists() ); // we served prefix file
- assertThat( "Proxy2 should have ES", proxy2EntrySource.exists() ); // we served prefix file
- assertThat( "Group should have ES", groupEntrySource.exists() ); // both proxies have it
+ assertThat( "Proxy1 should have ES", proxy1EntrySource.supported() ); // we served prefix file
+ assertThat( "Proxy2 should have ES", proxy2EntrySource.supported() ); // we served prefix file
+ assertThat( "Group should have ES", groupEntrySource.supported() ); // both proxies have it
// GROUP wl must have 4 entries: 1 from hosted (/com/sonatype) + 3 from proxied prefix file
final List<String> groupEntries = groupEntrySource.readEntries();
@@ -380,9 +380,9 @@ public void oneInvalidAndOneValidOutOfTwoPrefixFilesServed()
wm.getPrefixSourceFor( getRepositoryRegistry().getRepositoryWithFacet( GROUP_REPO_ID,
MavenRepository.class ) );
- assertThat( "Proxy1 should have ES", proxy1EntrySource.exists() ); // we served prefix file
- assertThat( "Proxy2 should not have ES", !proxy2EntrySource.exists() ); // we served invalid prefix file
- assertThat( "Group should not have ES", !groupEntrySource.exists() ); // both proxies have it
+ assertThat( "Proxy1 should have ES", proxy1EntrySource.supported() ); // we served prefix file
+ assertThat( "Proxy2 should not have ES", !proxy2EntrySource.supported() ); // we served invalid prefix file
+ assertThat( "Group should not have ES", !groupEntrySource.supported() ); // both proxies have it
final RoutingStatus status =
wm.getStatusFor( getRepositoryRegistry().getRepositoryWithFacet( PROXY2_REPO_ID,
View
4 nexus-core/src/test/java/org/sonatype/nexus/proxy/maven/routing/internal/RemotePrefixFileStrategyTest.java
@@ -224,7 +224,7 @@ public void discoverPlaintextPrefixFile()
equalTo( "Remote publishes prefix file (is less than a day old), using it." ) );
final PrefixSource entrySource = result.getPrefixSource();
- assertThat( entrySource.exists(), is( true ) );
+ assertThat( entrySource.supported(), is( true ) );
assertThat( entrySource.readEntries(), contains( "/org/apache/maven", "/org/sonatype", "/eu/flatwhite" ) );
assertThat( entrySource.readEntries().size(), equalTo( 3 ) );
}
@@ -265,7 +265,7 @@ public boolean execute( HttpServletRequest request, HttpServletResponse response
equalTo( "Remote publishes prefix file (is less than a day old), using it." ) );
final PrefixSource entrySource = result.getPrefixSource();
- assertThat( entrySource.exists(), is( true ) );
+ assertThat( entrySource.supported(), is( true ) );
assertThat( entrySource.readEntries(), contains( "/org/apache/maven", "/org/sonatype", "/eu/flatwhite" ) );
assertThat( entrySource.readEntries().size(), equalTo( 3 ) );
}
View
8 ...re/src/test/java/org/sonatype/nexus/proxy/maven/routing/internal/TextFilePrefixSourceMarshallerTest.java
@@ -134,7 +134,7 @@ public void roundtrip()
{
when( storageFileItem.getInputStream() ).thenReturn(
new ByteArrayInputStream( prefixFile1( true ).getBytes( UTF8 ) ) );
- final List<String> entries = m.read( storageFileItem );
+ final List<String> entries = m.read( storageFileItem ).entries();
assertThat( entries, is( notNullValue() ) );
assertThat( entries.size(), is( 3 ) );
@@ -153,7 +153,7 @@ public void roundtrip2()
// prefixFile2 is "find created" like, see CENTRAL-515
when( storageFileItem.getInputStream() ).thenReturn(
new ByteArrayInputStream( prefixFile2( true ).getBytes( UTF8 ) ) );
- final List<String> entries = m.read( storageFileItem );
+ final List<String> entries = m.read( storageFileItem ).entries();
assertThat( entries, is( notNullValue() ) );
assertThat( entries.size(), is( 3 ) );
@@ -175,7 +175,7 @@ public void roundtrip3()
// prefixFile2 is "find created" like, see CENTRAL-515
when( storageFileItem.getInputStream() ).thenReturn(
new ByteArrayInputStream( prefixFile3( true ).getBytes( UTF8 ) ) );
- final List<String> entries = m.read( storageFileItem );
+ final List<String> entries = m.read( storageFileItem ).entries();
assertThat( entries, is( notNullValue() ) );
assertThat( entries.size(), is( 1 ) );
@@ -197,7 +197,7 @@ public void roundtrip4()
// prefixFile2 is "find created" like, see CENTRAL-515
when( storageFileItem.getInputStream() ).thenReturn(
new ByteArrayInputStream( prefixFile4( true, false ).getBytes( UTF8 ) ) );
- final List<String> entries = m.read( storageFileItem );
+ final List<String> entries = m.read( storageFileItem ).entries();
assertThat( entries, is( notNullValue() ) );
assertThat( entries.size(), is( 3 ) );
View
6 ...core/src/test/java/org/sonatype/nexus/proxy/maven/routing/internal/WritablePrefixSourceModifierTest.java
@@ -264,6 +264,12 @@ public boolean exists()
}
@Override
+ public boolean supported()
+ {
+ return exists();
+ }
+
+ @Override
public List<String> readEntries()
throws IOException
{
View
25 nexus-test/nexus-core-testsuite/src/test/java/core/routing/ITSupport.java
@@ -17,6 +17,7 @@
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.notNullValue;
+import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
@@ -24,6 +25,9 @@
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.DefaultHttpClient;
+import org.sonatype.nexus.client.core.exception.NexusClientNotFoundException;
+import org.sonatype.nexus.client.core.subsystem.content.Location;
+import org.sonatype.nexus.client.core.subsystem.content.Content.Directive;
import org.sonatype.nexus.client.core.subsystem.routing.Routing;
import org.sonatype.nexus.testsuite.client.RoutingTest;
@@ -116,4 +120,25 @@ protected InputStream getPrefixFileFrom( final String url )
}
}
+ protected boolean exists( final Location location, Directive directive )
+ throws IOException
+ {
+ return content().existsWith( location, directive );
+ }
+
+ protected boolean noscrape( final Location location, Directive directive )
+ throws IOException
+ {
+ ByteArrayOutputStream buf = new ByteArrayOutputStream();
+ try
+ {
+ content().downloadWith( location, directive, buf );
+ return new String( buf.toByteArray(), "UTF-8" ).startsWith( "@ unsupported" );
+ }
+ catch ( NexusClientNotFoundException e )
+ {
+ return false; // requested file was not found, so the repository is not marked as no-scrape
+ }
+ }
+
}
View
24 nexus-test/nexus-core-testsuite/src/test/java/core/routing/WithGroupRepositoryIT.java
@@ -22,8 +22,8 @@
import org.junit.Test;
import org.junit.rules.Timeout;
import org.sonatype.nexus.client.core.exception.NexusClientBadRequestException;
-import org.sonatype.nexus.client.core.subsystem.content.Location;
import org.sonatype.nexus.client.core.subsystem.content.Content.Directive;
+import org.sonatype.nexus.client.core.subsystem.content.Location;
import org.sonatype.nexus.client.core.subsystem.routing.DiscoveryConfiguration;
import com.google.common.primitives.Ints;
@@ -59,12 +59,16 @@ public WithGroupRepositoryIT( final String nexusBundleCoordinates )
private final Location PREFIX_FILE_LOCATION = Location.repositoryLocation( REPO_ID, "/.meta/prefixes.txt" );
- private final Location NOSCRAPE_FILE_LOCATION = Location.repositoryLocation( REPO_ID, "/.meta/noscrape.txt" );
-
protected boolean exists( final Location location )
throws IOException
{
- return content().existsWith( location, Directive.GROUP_ONLY );
+ return exists( location, Directive.GROUP_ONLY );
+ }
+
+ protected boolean noscrape( final Location location )
+ throws IOException
+ {
+ return noscrape( location, Directive.GROUP_ONLY );
}
@Test
@@ -76,7 +80,7 @@ public void groupMustHaveWLPublishedWhenAllMembersHaveIt()
// waitForWLDiscoveryOutcome( "central" );
// waitForWLPublishingOutcomes( "central", REPO_ID );
assertThat( exists( PREFIX_FILE_LOCATION ), is( true ) );
- assertThat( exists( NOSCRAPE_FILE_LOCATION ), is( false ) );
+ assertThat( noscrape( PREFIX_FILE_LOCATION ), is( false ) );
}
@Test
@@ -84,7 +88,7 @@ public void groupLoosesWLIfMemberLooses()
throws Exception
{
assertThat( exists( PREFIX_FILE_LOCATION ), is( true ) );
- assertThat( exists( NOSCRAPE_FILE_LOCATION ), is( false ) );
+ assertThat( noscrape( PREFIX_FILE_LOCATION ), is( false ) );
{
final DiscoveryConfiguration config = routing().getDiscoveryConfigurationFor( "central" );
config.setEnabled( false );
@@ -93,8 +97,8 @@ public void groupLoosesWLIfMemberLooses()
// waitForWLDiscoveryOutcome( "central" );
// waitForWLPublishingOutcomes( "central", REPO_ID );
}
- assertThat( exists( PREFIX_FILE_LOCATION ), is( false ) );
- assertThat( exists( NOSCRAPE_FILE_LOCATION ), is( true ) );
+ assertThat( exists( PREFIX_FILE_LOCATION ), is( true ) );
+ assertThat( noscrape( PREFIX_FILE_LOCATION ), is( true ) );
{
final DiscoveryConfiguration config = routing().getDiscoveryConfigurationFor( "central" );
config.setEnabled( true );
@@ -104,7 +108,7 @@ public void groupLoosesWLIfMemberLooses()
// waitForWLPublishingOutcomes( "central", REPO_ID );
}
assertThat( exists( PREFIX_FILE_LOCATION ), is( true ) );
- assertThat( exists( NOSCRAPE_FILE_LOCATION ), is( false ) );
+ assertThat( noscrape( PREFIX_FILE_LOCATION ), is( false ) );
}
@Test( expected = NexusClientBadRequestException.class )
@@ -113,7 +117,7 @@ public void groupWLNotDeletable()
{
// we did no any waiting, e just booted nexus, so it must be present
assertThat( exists( PREFIX_FILE_LOCATION ), is( true ) );
- assertThat( exists( NOSCRAPE_FILE_LOCATION ), is( false ) );
+ assertThat( noscrape( PREFIX_FILE_LOCATION ), is( false ) );
content().delete( PREFIX_FILE_LOCATION );
}
}
View
24 nexus-test/nexus-core-testsuite/src/test/java/core/routing/WithHostedRepositoryIT.java
@@ -57,12 +57,16 @@ public WithHostedRepositoryIT( final String nexusBundleCoordinates )
private final Location PREFIX_FILE_LOCATION = Location.repositoryLocation( REPO_ID, "/.meta/prefixes.txt" );
- private final Location NOSCRAPE_FILE_LOCATION = Location.repositoryLocation( REPO_ID, "/.meta/noscrape.txt" );
-
protected boolean exists( final Location location )
throws IOException
{
- return content().exists( location );
+ return exists( location, null );
+ }
+
+ protected boolean noscrape( final Location location )
+ throws IOException
+ {
+ return noscrape( location, null );
}
@Test
@@ -71,7 +75,7 @@ public void hostedMustHaveWLPublishedRightOnBoot()
{
// we did no any waiting, e just booted nexus, so it must be present
assertThat( exists( PREFIX_FILE_LOCATION ), is( true ) );
- assertThat( exists( NOSCRAPE_FILE_LOCATION ), is( false ) );
+ assertThat( noscrape( PREFIX_FILE_LOCATION ), is( false ) );
}
/**
@@ -84,15 +88,15 @@ public void hostedWLRecreateOnReboot()
{
// we did no any waiting, e just booted nexus, so it must be present
assertThat( exists( PREFIX_FILE_LOCATION ), is( true ) );
- assertThat( exists( NOSCRAPE_FILE_LOCATION ), is( false ) );
+ assertThat( noscrape( PREFIX_FILE_LOCATION ), is( false ) );
content().delete( PREFIX_FILE_LOCATION );
assertThat( exists( PREFIX_FILE_LOCATION ), is( false ) );
- assertThat( exists( NOSCRAPE_FILE_LOCATION ), is( false ) );
+ assertThat( noscrape( PREFIX_FILE_LOCATION ), is( false ) );
// cannot reboot nexus, this below does not work
nexus().stop();
nexus().start();
assertThat( exists( PREFIX_FILE_LOCATION ), is( true ) );
- assertThat( exists( NOSCRAPE_FILE_LOCATION ), is( false ) );
+ assertThat( noscrape( PREFIX_FILE_LOCATION ), is( false ) );
}
@Test
@@ -101,13 +105,13 @@ public void hostedWLDeletableAndRecreateManually()
{
// we did no any waiting, e just booted nexus, so it must be present
assertThat( exists( PREFIX_FILE_LOCATION ), is( true ) );
- assertThat( exists( NOSCRAPE_FILE_LOCATION ), is( false ) );
+ assertThat( noscrape( PREFIX_FILE_LOCATION ), is( false ) );
content().delete( PREFIX_FILE_LOCATION );
assertThat( exists( PREFIX_FILE_LOCATION ), is( false ) );
- assertThat( exists( NOSCRAPE_FILE_LOCATION ), is( true ) );
+ assertThat( noscrape( PREFIX_FILE_LOCATION ), is( false ) ); // autorouting state is unknown
routing().updatePrefixFile( REPO_ID );
routingTest().waitForAllRoutingUpdateJobToStop();
assertThat( exists( PREFIX_FILE_LOCATION ), is( true ) );
- assertThat( exists( NOSCRAPE_FILE_LOCATION ), is( false ) );
+ assertThat( noscrape( PREFIX_FILE_LOCATION ), is( false ) );
}
}
View
28 nexus-test/nexus-core-testsuite/src/test/java/core/routing/WithProxyRepositoryIT.java
@@ -15,12 +15,14 @@
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.is;
+import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.concurrent.TimeUnit;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.Timeout;
+import org.sonatype.nexus.client.core.exception.NexusClientNotFoundException;
import org.sonatype.nexus.client.core.subsystem.content.Content.Directive;
import org.sonatype.nexus.client.core.subsystem.content.Location;
import org.sonatype.nexus.client.core.subsystem.routing.DiscoveryConfiguration;
@@ -58,12 +60,16 @@ public WithProxyRepositoryIT( final String nexusBundleCoordinates )
private final Location PREFIX_FILE_LOCATION = Location.repositoryLocation( REPO_ID, "/.meta/prefixes.txt" );
- private final Location NOSCRAPE_FILE_LOCATION = Location.repositoryLocation( REPO_ID, "/.meta/noscrape.txt" );
-
protected boolean exists( final Location location )
throws IOException
{
- return content().existsWith( location, Directive.LOCAL_ONLY );
+ return exists( location, Directive.LOCAL_ONLY );
+ }
+
+ protected boolean noscrape( final Location location )
+ throws IOException
+ {
+ return noscrape( location, Directive.LOCAL_ONLY );
}
@Test
@@ -74,7 +80,7 @@ public void proxyMustHaveWLPublishedWhenDiscoveryDone()
routingTest().waitForAllRoutingUpdateJobToStop();
// waitForWLDiscoveryOutcome( REPO_ID );
assertThat( exists( PREFIX_FILE_LOCATION ), is( true ) );
- assertThat( exists( NOSCRAPE_FILE_LOCATION ), is( false ) );
+ assertThat( noscrape( PREFIX_FILE_LOCATION ), is( false ) );
}
@Test
@@ -82,7 +88,7 @@ public void proxyLoosesWLIfDisabled()
throws Exception
{
assertThat( exists( PREFIX_FILE_LOCATION ), is( true ) );
- assertThat( exists( NOSCRAPE_FILE_LOCATION ), is( false ) );
+ assertThat( noscrape( PREFIX_FILE_LOCATION ), is( false ) );
{
final DiscoveryConfiguration config = routing().getDiscoveryConfigurationFor( REPO_ID );
config.setEnabled( false );
@@ -90,8 +96,8 @@ public void proxyLoosesWLIfDisabled()
routingTest().waitForAllRoutingUpdateJobToStop();
// waitForWLDiscoveryOutcome( REPO_ID );
}
- assertThat( exists( PREFIX_FILE_LOCATION ), is( false ) );
- assertThat( exists( NOSCRAPE_FILE_LOCATION ), is( true ) );
+ assertThat( exists( PREFIX_FILE_LOCATION ), is( true ) );
+ assertThat( noscrape( PREFIX_FILE_LOCATION ), is( true ) );
{
final DiscoveryConfiguration config = routing().getDiscoveryConfigurationFor( REPO_ID );
config.setEnabled( true );
@@ -100,7 +106,7 @@ public void proxyLoosesWLIfDisabled()
// waitForWLDiscoveryOutcome( REPO_ID );
}
assertThat( exists( PREFIX_FILE_LOCATION ), is( true ) );
- assertThat( exists( NOSCRAPE_FILE_LOCATION ), is( false ) );
+ assertThat( noscrape( PREFIX_FILE_LOCATION ), is( false ) );
}
@Test
@@ -109,16 +115,16 @@ public void proxyWLDeletableAndRecreateManually()
{
// we did no any waiting, e just booted nexus, so it must be present
assertThat( exists( PREFIX_FILE_LOCATION ), is( true ) );
- assertThat( exists( NOSCRAPE_FILE_LOCATION ), is( false ) );
+ assertThat( noscrape( PREFIX_FILE_LOCATION ), is( false ) );
content().delete( PREFIX_FILE_LOCATION );
routingTest().waitForAllRoutingUpdateJobToStop();
// waitForWLDiscoveryOutcome( REPO_ID );
assertThat( exists( PREFIX_FILE_LOCATION ), is( false ) );
- assertThat( exists( NOSCRAPE_FILE_LOCATION ), is( true ) );
+ assertThat( noscrape( PREFIX_FILE_LOCATION ), is( false ) );
routing().updatePrefixFile( REPO_ID );
routingTest().waitForAllRoutingUpdateJobToStop();
// waitForWLDiscoveryOutcome( REPO_ID );
assertThat( exists( PREFIX_FILE_LOCATION ), is( true ) );
- assertThat( exists( NOSCRAPE_FILE_LOCATION ), is( false ) );
+ assertThat( noscrape( PREFIX_FILE_LOCATION ), is( false ) );
}
}
Please sign in to comment.
Something went wrong with that request. Please try again.