Permalink
Browse files

Cleanup: reconciled autorouting startup and periodic update logic

Automatic routing periodic update logic is now able to initialize
hosted repository prefix path list as needed. No need to do this
explicitly during startup any more.

Signed-off-by: Igor Fedorenko <igor@ifedorenko.com>
  • Loading branch information...
1 parent 2d29469 commit edeaa1fcbc48b125c4c943d2abd30f60d9f6a1f2 @ifedorenko ifedorenko committed Mar 30, 2013
@@ -16,16 +16,17 @@
import java.util.List;
/**
- * Prefix source main purpose is to serve/read-up WL prefix entries. It might be backed (or sourced) from multiple things, like
- * file or even a plain array of strings.
+ * Prefix source main purpose is to serve/read-up WL prefix entries. It might be backed (or sourced) from multiple
+ * things, like file or even a plain array of strings.
*
* @author cstamas
* @since 2.4
*/
public interface PrefixSource
{
/**
- * Returns {@code true} if this entry source exists, hence, is readable.
+ * Returns {@code true} if this entry source exists, hence, is readable. Behaviour of other PrefixSource methods is
+ * undefined if the source does not exist, i.e. this method returns {@code false}.
*
* @return {@code true} if entry source exists, {@code false} otherwise.
*/
@@ -213,11 +213,6 @@ public Long call()
public void writeEntries( final PrefixSource prefixSource )
throws IOException
{
- if ( prefixSource instanceof FilePrefixSource && equals( (FilePrefixSource) prefixSource ) )
- {
- // we would read and then write to the same file, don't do it
- return;
- }
final ByteArrayOutputStream bos = new ByteArrayOutputStream();
@cstamas

cstamas Apr 2, 2013

Contributor

This change causes now that Nexus modifies the prefix file downloaded from remote, something it should not do. By specs, the prefix file we got from remote must publish downstream in unchanged form (as we got it).

Take for example Central prefix file, compare the "original":
http://repo1.maven.org/maven2/.meta/prefixes.txt
with the one published by 07 bundle:
http://localhost:8081/nexus/content/repositories/central/.meta/prefixes.txt
(latter will have "generated" headers added).

Am not saying this breaks anything (does not), just emphasising that we had a requirement for it in feature.

@peterlynch

peterlynch Apr 2, 2013

Member

If a test didn't break after this requirement was broken - then we are certainly missing a test for it.

getPrefixSourceMarshaller().write( prefixSource.readEntries(), bos );
putFileItem( new PreparedContentLocator( new ByteArrayInputStream( bos.toByteArray() ), "text/plain" ) );
@@ -124,12 +124,6 @@
private final ConstrainedExecutor constrainedExecutor;
/**
- * Flag that marks that "boot sequence" is finished. If {@code true}, all the "boot initialization" already
- * happened. Still, depending on {@link WLConfig#isFeatureActive()} what actually was done during this.
- */
- private volatile boolean bootSequenceDone;
-
- /**
* Da constructor.
*
* @param eventBus
@@ -169,118 +163,54 @@ public ManagerImpl( final EventBus eventBus, final ApplicationStatusSource appli
@Override
public void startup()
{
- bootSequenceDone = false;
-
if ( config.isFeatureActive() )
{
- // init WLs of repositories on boot. In first "flight" we do not do any update
- // only ack existing WLs or in case of non-existent (upgrade), we just mark those
- // reposes as noscrape. First the hosted+proxy reposes are processed, and they
- // are gathered into a list that we know they need-update (have no WL at all)
- // 2nd pass is for groups, but they are NOT collected for updates.
- // Finally, those collected for update will get update bg jobs spawned.
-
- // All this is important for 1st boot only, as on subsequent boot WLs will be already
- // present and just event will be published.
- // hosted + proxies get inited first, collect those needing update
- // those will be all on upgrade, and none on subsequent boots
- final ArrayList<MavenRepository> needUpdateRepositories = new ArrayList<MavenRepository>();
+ // Send events about repositories with existing PrefixSource synchronously as part of #startup method
+ // This allows proper initialization of components that need to track state of automatic routing
+ for ( MavenRepository mavenRepository : repositoryRegistry.getRepositoriesWithFacet( MavenRepository.class ) )
{
- final ArrayList<MavenRepository> initableRepositories = new ArrayList<MavenRepository>();
- initableRepositories.addAll( repositoryRegistry.getRepositoriesWithFacet( MavenHostedRepository.class ) );
- initableRepositories.addAll( repositoryRegistry.getRepositoriesWithFacet( MavenProxyRepository.class ) );
- for ( MavenRepository mavenRepository : initableRepositories )
+ if ( isMavenRepositorySupported( mavenRepository )
+ && mavenRepository.getLocalStatus().shouldServiceRequest() )
{
- if ( isMavenRepositorySupported( mavenRepository )
- && mavenRepository.getLocalStatus().shouldServiceRequest() )
+ final FilePrefixSource prefixSource = getPrefixSourceFor( mavenRepository );
+ if ( prefixSource.exists() )
{
- if ( doInitializePrefixFileOnStartup( mavenRepository ) )
+ getLogger().debug( "Initializing prefix file of {}", mavenRepository );
+ if ( prefixSource.supported() )
{
- // collect those marked as need-update
- needUpdateRepositories.add( mavenRepository );
+ eventBus.post( new PrefixFilePublishedRepositoryEvent( mavenRepository, prefixSource ) );
+ }
+ else
+ {
+ eventBus.post( new PrefixFileUnpublishedRepositoryEvent( mavenRepository ) );
}
- }
- }
- }
- // groups get inited next, this mostly means they will be marked as noscrape on upgraded instances,
- // and just a published event will be fired on consequent boots
- {
- final ArrayList<MavenRepository> initableGroupRepositories = new ArrayList<MavenRepository>();
- initableGroupRepositories.addAll( repositoryRegistry.getRepositoriesWithFacet( MavenGroupRepository.class ) );
- for ( MavenRepository mavenRepository : initableGroupRepositories )
- {
- if ( isMavenRepositorySupported( mavenRepository )
- && mavenRepository.getLocalStatus().shouldServiceRequest() )
- {
- // groups will not be collected to needs-update list
- doInitializePrefixFileOnStartup( mavenRepository );
}
}
}
- // schedule runnable to perform boot-sequence, that as last step schedules updater that ping hourly the
- // mayUpdateProxyWhitelist method
- // boot-sequence will run and handle on-boot-updates for repositories collected into list above.
- // This all happens in periodicUpdater to not defer boot sequence in case
- // when upgrade happens on larger instance (as then potentially many hosted/proxy reposes
- // will need prefix file to be built)
- executor.execute( new Runnable()
+ executor.scheduleAtFixedRate( new Runnable()
{
@Override
public void run()
{
- // perform possible updates, create prefix files for hosted and proxy reposes
- try
+ if ( !applicationStatusSource.getSystemStatus().isNexusStarted() )
{
- for ( MavenRepository mavenRepository : needUpdateRepositories )
- {
- try
- {
- updatePrefixFile( mavenRepository );
- }
- catch ( Exception e )
- {
- getLogger().warn( "Could not perform initial WL build or repository {}",
- mavenRepository, e );
- }
- }
-
- // after that, schedule a new "updater" runnable that with
- // periodically perform updates to proxy reposes only
- executor.scheduleAtFixedRate( new Runnable()
- {
- @Override
- public void run()
- {
- if ( !applicationStatusSource.getSystemStatus().isNexusStarted() )
- {
- // this might happen on periodic call AFTER nexus shutdown was commenced
- // or BEFORE nexus booted, if some other plugin/subsystem delays boot for some
- // reason.
- // None of those is a problem, in latter case we will do what we need in next tick.
- // In former case,
- // we should not do anything anyway, we are being shut down.
- getLogger().debug( "Nexus not yet started, bailing out" );
- return;
- }
- mayUpdateAllProxyPrefixFiles();
- }
- }, TimeUnit.HOURS.toMillis( 1 ), TimeUnit.HOURS.toMillis( 1 ), TimeUnit.MILLISECONDS );
- }
- finally
- {
- bootSequenceDone = true;
+ // this might happen on periodic call AFTER nexus shutdown was commenced
+ // or BEFORE nexus booted, if some other plugin/subsystem delays boot for some
+ // reason.
+ // None of those is a problem, in latter case we will do what we need in next tick.
+ // In former case,
+ // we should not do anything anyway, we are being shut down.
+ getLogger().debug( "Nexus not yet started, bailing out" );
+ return;
}
+ mayUpdateAllProxyPrefixFiles();
}
- } );
+ }, 0L /*no initial delay*/, TimeUnit.HOURS.toMillis( 1 ), TimeUnit.MILLISECONDS );
// register event dispatcher, to start receiving events
eventBus.register( eventDispatcher );
}
- else
- {
- bootSequenceDone = true;
- }
}
@Override
@@ -332,77 +262,42 @@ public void initializePrefixFile( final MavenRepository mavenRepository )
}
/**
- * Initializes maven repository WL on startup. Signals with returning {@code true} if the repository needs update of
- * WL.
- *
- * @param mavenRepository
- * @return {@code true} if repository needs update.
- */
- protected boolean doInitializePrefixFileOnStartup( final MavenRepository mavenRepository )
- {
- getLogger().debug( "Initializing prefix file of {}", mavenRepository );
- final PrefixSource prefixSource = getPrefixSourceFor( mavenRepository );
- try
- {
- if ( prefixSource.supported() )
- {
- // good, we assume is up to date, which should be unless user tampered with it
- // in that case, just delete it + update and should be fixed.
- publish( mavenRepository, prefixSource, false );
- getLogger().info( "Existing prefix file of {} initialized",
- RepositoryStringUtils.getHumanizedNameString( mavenRepository ) );
- }
- else
- {
- // mark it for noscrape if not marked yet
- // this is mainly important on 1st boot or newly added reposes
- unpublish( mavenRepository, false );
- getLogger().info( "Initializing non-existing prefix file of {}",
- RepositoryStringUtils.getHumanizedNameString( mavenRepository ) );
- return true;
- }
- }
- catch ( Exception e )
- {
- getLogger().warn( "Problem during prefix file initialisation of {}",
- RepositoryStringUtils.getHumanizedNameString( mavenRepository ), e );
- try
- {
- unpublish( mavenRepository, false );
- }
- catch ( IOException ioe )
- {
- // silently
- }
- }
- return false;
- }
-
- /**
* Method meant to be invoked on regular periods (like hourly, as we defined "resolution" of WL update period in
* hours too), and will perform WL update only on those proxy repositories that needs it.
*/
protected void mayUpdateAllProxyPrefixFiles()
{
getLogger().trace( "mayUpdateAllProxyPrefixFiles started" );
- final List<MavenProxyRepository> mavenProxyRepositories =
- repositoryRegistry.getRepositoriesWithFacet( MavenProxyRepository.class );
- for ( MavenProxyRepository mavenProxyRepository : mavenProxyRepositories )
+ for ( MavenRepository mavenRepository : repositoryRegistry.getRepositoriesWithFacet( MavenRepository.class ) )
{
try
{
- mayUpdateProxyPrefixFile( mavenProxyRepository );
+ final FilePrefixSource prefixSource = getPrefixSourceFor( mavenRepository );
+ if ( !prefixSource.exists() )
+ {
+ // automatic routing has not been initialized for this repository yet, for initialization.
+ doUpdatePrefixFileAsync( true, mavenRepository );
+ }
+ else
+ {
+ MavenProxyRepository mavenProxyRepository =
+ mavenRepository.adaptToFacet( MavenProxyRepository.class );
+ if ( mavenProxyRepository != null )
+ {
+ mayUpdateProxyPrefixFile( mavenProxyRepository );
+ }
+ }
}
catch ( IllegalStateException e )
{
// just neglect it and continue, this one might be auto blocked if proxy or put out of service
- getLogger().trace( "Proxy repository {} is not in state to be updated", mavenProxyRepository );
+ getLogger().trace( "Repository {} is not in state to be updated", mavenRepository );
}
catch ( Exception e )
{
// just neglect it and continue, but do log it
- getLogger().warn( "Problem during prefix file update of proxy repository {}",
- RepositoryStringUtils.getHumanizedNameString( mavenProxyRepository ), e );
+ getLogger().warn( "Problem during prefix file update of repository {}",
+ RepositoryStringUtils.getHumanizedNameString( mavenRepository ), e );
}
}
}
@@ -608,11 +503,6 @@ protected boolean doUpdatePrefixFileAsync( final boolean forced, final MavenRepo
@VisibleForTesting
public boolean isUpdatePrefixFileJobRunning()
{
- if ( !bootSequenceDone )
- {
- getLogger().debug( "Boot update sequence not done yet" );
- return true;
- }
final Statistics statistics = constrainedExecutor.getStatistics();
getLogger().debug( "Running update jobs for {}", statistics.getCurrentlyRunningJobKeys() );
return !statistics.getCurrentlyRunningJobKeys().isEmpty();
@@ -1086,13 +976,6 @@ public boolean revokeEntry( final MavenHostedRepository mavenHostedRepository, f
public void publish( final MavenRepository mavenRepository, final PrefixSource prefixSource )
throws IOException
{
- publish( mavenRepository, prefixSource, true );
- }
-
- protected void publish( final MavenRepository mavenRepository, final PrefixSource prefixSource,
- final boolean propagate )
- throws IOException
- {
// publish prefix file
final FilePrefixSource prefixesFile = getPrefixSourceFor( mavenRepository );
try
@@ -1108,33 +991,21 @@ protected void publish( final MavenRepository mavenRepository, final PrefixSourc
// event
eventBus.post( new PrefixFilePublishedRepositoryEvent( mavenRepository, prefixesFile ) );
- if ( propagate )
- {
- // propagate
- propagatePrefixFileUpdateOf( mavenRepository );
- }
+ // propagate
+ propagatePrefixFileUpdateOf( mavenRepository );
}
@Override
public void unpublish( final MavenRepository mavenRepository )
throws IOException
{
- unpublish( mavenRepository, true );
- }
-
- protected void unpublish( final MavenRepository mavenRepository, final boolean propagate )
- throws IOException
- {
getPrefixSourceFor( mavenRepository ).writeUnsupported();
// event
eventBus.post( new PrefixFileUnpublishedRepositoryEvent( mavenRepository ) );
- if ( propagate )
- {
- // propagate
- propagatePrefixFileUpdateOf( mavenRepository );
- }
+ // propagate
+ propagatePrefixFileUpdateOf( mavenRepository );
}
protected void propagatePrefixFileUpdateOf( final MavenRepository mavenRepository )
@@ -1146,8 +1017,8 @@ protected void propagatePrefixFileUpdateOf( final MavenRepository mavenRepositor
containingGroupRepository = groupRepository.adaptToFacet( MavenGroupRepository.class );
if ( mavenRepository != null )
{
- // this method is invoked while holding write lock on mavenRepository
- // group prefix file will need read lock on all members
+ // this method is invoked while holding write lock on mavenRepository prefix file
+ // groupRepository prefix file calculation will need read locks on all members prefix files
// to avoid deadlocks we push group prefix file update to another thread
doUpdatePrefixFileAsync( true, containingGroupRepository );
}

0 comments on commit edeaa1f

Please sign in to comment.