Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

NEXUS-5602: Using the periodic update thread for initial updates too

On 1st boot of updated instances, a lot of MavenRepository
instances (hosted and proxy) will be piled up in the array
needUpdateRepositories in WLManagerImpl#startup().

Currently, as the thread pool with fixed size of 5 and
blocking queue was deferring update.

With this change, the for-loop doing initial WL build
(walk of the content for hosted and discovery for proxies)
is deferred, and periodic updater thread is used.

This also means, that upon boot, the initial delay of
first run will be applied, and on that moment on, hosted
reposes will have WL, not before that.

Initial change breaks all the UTs and ITs that uses
exposed-for-tests-only method to reach a state when
the test should start.

isUpdateWhitelistJobRunning() method used for this
is now updated, and will flag (and make tests wait)
as long as boot sequence is done (newly added) AND
there are no bg updates running (as before).

Also, EventDispatcher simplified, the feature active
flag is not passed anymore, instead, it is not
registered to event bus when feature is not
active, resulting in same behavior (before, when
feature was inactive EventDispatcher got events
but simply did not invoke WL Manager).
  • Loading branch information...
commit 4a401a58d02390afb5a4353b22bd6cbe41f7bd90 1 parent 5f6c453
@cstamas cstamas authored
View
16 nexus-core/src/main/java/org/sonatype/nexus/proxy/maven/routing/internal/EventDispatcher.java
@@ -58,19 +58,15 @@
private final Manager wlManager;
- private final boolean active;
-
/**
* Da constructor.
*
* @param wlManager
- * @param active
*/
- public EventDispatcher( final Manager wlManager, final boolean active )
+ public EventDispatcher( final Manager wlManager )
{
this.logger = LoggerFactory.getLogger( getClass() );
this.wlManager = checkNotNull( wlManager );
- this.active = active;
}
protected Logger getLogger()
@@ -156,11 +152,6 @@ protected void revokePath( final MavenHostedRepository mavenHostedRepository, St
// == Filters
- protected boolean isActive()
- {
- return active;
- }
-
protected boolean isRequestContextMarked( final RequestContext context )
{
return context.containsKey( Manager.ROUTING_INITIATED_FILE_OPERATION_FLAG_KEY );
@@ -168,10 +159,9 @@ protected boolean isRequestContextMarked( final RequestContext context )
protected boolean isRepositoryHandled( final Repository repository )
{
- // we handle repository events after this isActive, is not out of service, and only for non-shadow repository
+ // we handle repository events if repo is not out of service, and only for non-shadow repository
// that are Maven2 reposes
- return isActive() && repository != null
- && repository.getRepositoryKind().isFacetAvailable( MavenRepository.class )
+ return repository != null && repository.getRepositoryKind().isFacetAvailable( MavenRepository.class )
&& !repository.getRepositoryKind().isFacetAvailable( ShadowRepository.class )
&& Maven2ContentClass.ID.equals( repository.getRepositoryContentClass().getId() );
}
View
162 nexus-core/src/main/java/org/sonatype/nexus/proxy/maven/routing/internal/ManagerImpl.java
@@ -130,6 +130,12 @@
private final ConstrainedExecutor constrainedExecutor;
/**
+ * Flag that marks that "boot sequence" is finished. If {@code true}, all the "boot initialization" already
+ * happened. Still, depending on {@link WLConfig#isFeatureActive()} what actually was done during this.
+ */
+ private volatile boolean bootSequenceDone;
+
+ /**
* Da constructor.
*
* @param eventBus
@@ -162,76 +168,125 @@ public ManagerImpl( final EventBus eventBus, final ApplicationStatusSource appli
new ThreadPoolExecutor.AbortPolicy() );
this.constrainedExecutor = new ConstrainedExecutorImpl( executor );
// register event dispatcher
- this.eventDispatcher = new EventDispatcher( this, config.isFeatureActive() );
+ this.eventDispatcher = new EventDispatcher( this );
this.eventBus.register( this );
}
@Override
public void startup()
{
- // init WLs of repositories on boot. In first "flight" we do not do any update
- // only ack existing WLs or in case of non-existent (upgrade), we just mark those
- // reposes as noscrape. First the hosted+proxy reposes are processed, and they
- // are gathered into a list that we know they need-update (have no WL at all)
- // 2nd pass is for groups, but they are NOT collected for updates.
- // Finally, those collected for update will get update bg jobs spawned.
-
- // All this is important for 1st boot only, as on subsequent boot WLs will be already
- // present and just event will be published.
- // hosted + proxies get inited first, collect those needing update
- // those will be all on upgrade, and none on subsequent boots
- final ArrayList<MavenRepository> needUpdateRepositories = new ArrayList<MavenRepository>();
+ bootSequenceDone = false;
+
+ if ( config.isFeatureActive() )
{
- final ArrayList<MavenRepository> initableRepositories = new ArrayList<MavenRepository>();
- initableRepositories.addAll( repositoryRegistry.getRepositoriesWithFacet( MavenHostedRepository.class ) );
- initableRepositories.addAll( repositoryRegistry.getRepositoriesWithFacet( MavenProxyRepository.class ) );
- for ( MavenRepository mavenRepository : initableRepositories )
+ // init WLs of repositories on boot. In first "flight" we do not do any update
+ // only ack existing WLs or in case of non-existent (upgrade), we just mark those
+ // reposes as noscrape. First the hosted+proxy reposes are processed, and they
+ // are gathered into a list that we know they need-update (have no WL at all)
+ // 2nd pass is for groups, but they are NOT collected for updates.
+ // Finally, those collected for update will get update bg jobs spawned.
+
+ // All this is important for 1st boot only, as on subsequent boot WLs will be already
+ // present and just event will be published.
+ // hosted + proxies get inited first, collect those needing update
+ // those will be all on upgrade, and none on subsequent boots
+ final ArrayList<MavenRepository> needUpdateRepositories = new ArrayList<MavenRepository>();
{
- if ( isMavenRepositorySupported( mavenRepository )
- && mavenRepository.getLocalStatus().shouldServiceRequest() )
+ final ArrayList<MavenRepository> initableRepositories = new ArrayList<MavenRepository>();
+ initableRepositories.addAll( repositoryRegistry.getRepositoriesWithFacet( MavenHostedRepository.class ) );
+ initableRepositories.addAll( repositoryRegistry.getRepositoriesWithFacet( MavenProxyRepository.class ) );
+ for ( MavenRepository mavenRepository : initableRepositories )
{
- if ( doInitializePrefixFileOnStartup( mavenRepository ) )
+ if ( isMavenRepositorySupported( mavenRepository )
+ && mavenRepository.getLocalStatus().shouldServiceRequest() )
{
- // collect those marked as need-update
- needUpdateRepositories.add( mavenRepository );
+ if ( doInitializePrefixFileOnStartup( mavenRepository ) )
+ {
+ // collect those marked as need-update
+ needUpdateRepositories.add( mavenRepository );
+ }
}
}
}
- }
- // groups get inited next, this mostly means they will be marked as noscrape on upgraded instances,
- // and just a published event will be fired on consequent boots
- {
- final ArrayList<MavenRepository> initableGroupRepositories = new ArrayList<MavenRepository>();
- initableGroupRepositories.addAll( repositoryRegistry.getRepositoriesWithFacet( MavenGroupRepository.class ) );
- for ( MavenRepository mavenRepository : initableGroupRepositories )
+ // groups get inited next, this mostly means they will be marked as noscrape on upgraded instances,
+ // and just a published event will be fired on consequent boots
{
- if ( isMavenRepositorySupported( mavenRepository )
- && mavenRepository.getLocalStatus().shouldServiceRequest() )
+ final ArrayList<MavenRepository> initableGroupRepositories = new ArrayList<MavenRepository>();
+ initableGroupRepositories.addAll( repositoryRegistry.getRepositoriesWithFacet( MavenGroupRepository.class ) );
+ for ( MavenRepository mavenRepository : initableGroupRepositories )
{
- // groups will not be collected to needs-update list
- doInitializePrefixFileOnStartup( mavenRepository );
+ if ( isMavenRepositorySupported( mavenRepository )
+ && mavenRepository.getLocalStatus().shouldServiceRequest() )
+ {
+ // groups will not be collected to needs-update list
+ doInitializePrefixFileOnStartup( mavenRepository );
+ }
}
}
+
+ // schedule runnable to perform boot-sequence, that as last step schedules updater that ping hourly the
+ // mayUpdateProxyWhitelist method
+ // boot-sequence will run and handle on-boot-updates for repositories collected into list above.
+ // This all happens in periodicUpdater to not defer boot sequence in case
+ // when upgrade happens on larger instance (as then potentially many hosted/proxy reposes
+ // will need prefix file to be built)
+ executor.execute( new Runnable()
+ {
+ @Override
+ public void run()
+ {
+ // perform possible updates, create prefix files for hosted and proxy reposes
+ try
+ {
+ for ( MavenRepository mavenRepository : needUpdateRepositories )
+ {
+ try
+ {
+ updatePrefixFile( mavenRepository );
+ }
+ catch ( Exception e )
+ {
+ getLogger().warn( "Could not perform initial WL build or repository {}",
+ mavenRepository, e );
+ }
+ }
+
+ // after that, schedule a new "updater" runnable that with
+ // periodically perform updates to proxy reposes only
+ executor.scheduleAtFixedRate( new Runnable()
+ {
+ @Override
+ public void run()
+ {
+ if ( !applicationStatusSource.getSystemStatus().isNexusStarted() )
+ {
+ // this might happen on periodic call AFTER nexus shutdown was commenced
+ // or BEFORE nexus booted, if some other plugin/subsystem delays boot for some
+ // reason.
+ // None of those is a problem, in latter case we will do what we need in next tick.
+ // In former case,
+ // we should not do anything anyway, we are being shut down.
+ getLogger().debug( "Nexus not yet started, bailing out" );
+ return;
+ }
+ mayUpdateAllProxyPrefixFiles();
+ }
+ }, TimeUnit.HOURS.toMillis( 1 ), TimeUnit.HOURS.toMillis( 1 ), TimeUnit.MILLISECONDS );
+ }
+ finally
+ {
+ bootSequenceDone = true;
+ }
+ }
+ } );
+
+ // register event dispatcher, to start receiving events
+ eventBus.register( eventDispatcher );
}
- // spawn all the needed updates as bg jobs
- // these will maintaing groups too as needed
- for ( MavenRepository mavenRepository : needUpdateRepositories )
+ else
{
- updatePrefixFile( mavenRepository );
+ bootSequenceDone = true;
}
-
- // schedule the "updater" that ping hourly the mayUpdateAllProxyPrefixFiles method
- // but wait 1 minute for boot to calm down and then start
- this.executor.scheduleAtFixedRate( new Runnable()
- {
- @Override
- public void run()
- {
- mayUpdateAllProxyPrefixFiles();
- }
- }, TimeUnit.MINUTES.toMillis( 1 ), TimeUnit.HOURS.toMillis( 1 ), TimeUnit.MILLISECONDS );
- // register event dispatcher, to start receiving events
- eventBus.register( eventDispatcher );
}
@Override
@@ -554,11 +609,16 @@ protected boolean doUpdatePrefixFileAsync( final boolean forced, final MavenRepo
* Is visible to expose over the nexus-it-helper-plugin only, and UTs are using this. Should not be used for other
* means.
*
- * @return {@code true} if there are prefix file update jobs running.
+ * @return {@code true} if there are prefix file update jobs running, or boot of feature not yet finished.
*/
@VisibleForTesting
public boolean isUpdatePrefixFileJobRunning()
{
+ if ( !bootSequenceDone )
+ {
+ getLogger().debug( "Boot update sequence not done yet" );
+ return true;
+ }
final Statistics statistics = constrainedExecutor.getStatistics();
getLogger().debug( "Running update jobs for {}", statistics.getCurrentlyRunningJobKeys() );
return !statistics.getCurrentlyRunningJobKeys().isEmpty();
View
3  nexus-test/nexus-core-testsuite/src/test/java/core/routing/DisabledSmokeIT.java
@@ -76,6 +76,7 @@ public void checkPublicGroupResponse()
// public
final Status publicStatus = routing().getStatus( "public" );
assertThat( publicStatus.getPublishedStatus(), equalTo( Outcome.FAILED ) );
+ assertThat( publicStatus.getDiscoveryStatus(), is( nullValue() ) );
}
@Test
@@ -83,7 +84,7 @@ public void checkReleasesHostedResponse()
{
// releases
final Status releasesStatus = routing().getStatus( "releases" );
- assertThat( releasesStatus.getPublishedStatus(), equalTo( Outcome.SUCCEEDED ) );
+ assertThat( releasesStatus.getPublishedStatus(), equalTo( Outcome.FAILED ) );
assertThat( releasesStatus.getDiscoveryStatus(), is( nullValue() ) );
}
View
26 ...et1x/nexus-restlet1x-plugin/src/test/java/org/sonatype/nexus/rest/routing/RoutingStatusResourceTest.java
@@ -26,7 +26,11 @@
import org.restlet.data.Reference;
import org.restlet.data.Request;
import org.restlet.data.Response;
+import org.sonatype.nexus.ApplicationStatusSource;
import org.sonatype.nexus.NexusAppTestSupport;
+import org.sonatype.nexus.SystemState;
+import org.sonatype.nexus.proxy.maven.routing.Manager;
+import org.sonatype.nexus.proxy.maven.routing.internal.ManagerImpl;
import org.sonatype.nexus.proxy.registry.RepositoryRegistry;
import org.sonatype.nexus.proxy.repository.Repository;
import org.sonatype.nexus.rest.model.RoutingStatusMessageWrapper;
@@ -43,6 +47,7 @@
public void login()
throws Exception
{
+ lookup( ApplicationStatusSource.class ).setState( SystemState.STARTED );
ThreadContext.bind( new Subject.Builder().buildSubject() );
}
@@ -53,6 +58,23 @@ public void logout()
ThreadContext.remove();
}
+ @Override
+ protected boolean enableAutomaticRoutingFeature()
+ {
+ return true;
+ }
+
+ protected void waitForRoutingBackgroundUpdates()
+ throws Exception
+ {
+ // TODO: A hack, I don't want to expose this over component contract iface
+ final ManagerImpl wm = (ManagerImpl) lookup( Manager.class );
+ while ( wm.isUpdatePrefixFileJobRunning() )
+ {
+ Thread.sleep( 500 );
+ }
+ }
+
/**
* Testing does {@link RoutingStatusResource} honors {@link Repository#isExposed()} flag, since it has to be handled at
* REST level as it exactly prevents access to repository over HTTP layer. Internally, exposed repositories are
@@ -64,6 +86,10 @@ public void logout()
public void statusUrlHonorsRepoState()
throws Exception
{
+ final Manager manager = lookup( Manager.class );
+ manager.startup();
+ waitForRoutingBackgroundUpdates();
+
final RoutingStatusResource wlStatusResource = (RoutingStatusResource) lookup( PlexusResource.class, "RoutingStatusResource" );
final Request request = new Request();
request.setRootRef( new Reference( "http://localhost:8081/nexus" ) );
Please sign in to comment.
Something went wrong with that request. Please try again.