Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -511,4 +511,5 @@ protected Set<Group> affectedByFromStores( final Collection<StoreKey> keys )

return groups;
}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
package org.commonjava.indy.infinispan.data;

import org.commonjava.indy.action.IndyLifecycleException;
import org.commonjava.indy.action.StartupAction;
import org.commonjava.indy.data.StoreDataManager;

import javax.inject.Inject;

public class InfinispanStoreDataByPkgMapStartupAction
implements StartupAction
{

@Inject
private StoreDataManager storeDataManager;

@Override
public void start()
throws IndyLifecycleException
{
if ( storeDataManager instanceof InfinispanStoreDataManager )
{
((InfinispanStoreDataManager)storeDataManager).initByPkgMap();
}
}

@Override
public int getStartupPriority()
{
return 11;
}

@Override
public String getId()
{
return "Infinispan by-pkg map initializer";
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
package org.commonjava.indy.infinispan.data;

import org.commonjava.indy.audit.ChangeSummary;
import org.commonjava.indy.data.IndyDataException;
import org.commonjava.indy.data.NoOpStoreEventDispatcher;
import org.commonjava.indy.data.StoreEventDispatcher;
import org.commonjava.indy.db.common.AbstractStoreDataManager;
Expand Down Expand Up @@ -84,37 +85,6 @@ protected InfinispanStoreDataManager()
{
}

@PostConstruct
synchronized void init()
{
// re-fill the stores by package cache each time when reboot
if ( storesByPkg != null )
{
logger.info( "Clean the stores-by-pkg cache" );
storesByPkg.clear();
}

boolean affectedByIsEmpty = affectedByStores.isEmpty();

final Set<ArtifactStore> allStores = getAllArtifactStores();
logger.info( "There are {} stores need to fill in stores-by-pkg cache", allStores.size() );
for ( ArtifactStore store : allStores )
{
final Map<StoreType, Set<StoreKey>> typedKeys =
storesByPkg.computeIfAbsent( store.getKey().getPackageType(), k -> new HashMap<>() );

final Set<StoreKey> keys = typedKeys.computeIfAbsent( store.getKey().getType(), k -> new HashSet<>() );
keys.add( store.getKey() );

// If the affected-by cache is empty AND this is a group, record the reverse-map of constituents.
if ( affectedByIsEmpty && store.getType() == group )
{
new HashSet<>( ( (Group) store ).getConstituents() ).forEach(
key -> affectedByStores.computeIfAbsent( key, k -> new HashSet<>() ).add( store.getKey() ) );
}
}
}

public InfinispanStoreDataManager( final Cache<StoreKey, ArtifactStore> cache,
final Cache<String, Map<StoreType, Set<StoreKey>>> storesByPkg,
final Cache<StoreKey, Set<StoreKey>> affectedByStoresCache )
Expand All @@ -123,7 +93,7 @@ public InfinispanStoreDataManager( final Cache<StoreKey, ArtifactStore> cache,
this.stores = new CacheHandle( STORE_DATA_CACHE, cache );
this.storesByPkg = new CacheHandle( STORE_BY_PKG_CACHE, storesByPkg );
this.affectedByStores = new CacheHandle( AFFECTED_BY_STORE_CACHE, affectedByStoresCache );
init();
logger.warn( "Constructor init: STARTUP ACTIONS MAY NOT RUN." );
}

@Override
Expand Down Expand Up @@ -251,6 +221,8 @@ public Set<StoreKey> getStoreKeysByPkgAndType( final String pkg, final StoreType
@Override
public Set<Group> affectedBy( final Collection<StoreKey> keys )
{
checkAffectedByCacheHealth();

final Set<Group> result = new HashSet<>();

// use these to avoid recursion
Expand Down Expand Up @@ -295,6 +267,9 @@ public Set<Group> affectedBy( final Collection<StoreKey> keys )
}
}

logger.info( "Groups affected by {} are: {}", keys,
result.stream().map( ArtifactStore::getKey ).collect( Collectors.toSet() ) );

return result;
}

Expand All @@ -310,9 +285,12 @@ protected void refreshAffectedBy( final ArtifactStore store, final ArtifactStore
{
if ( store instanceof Group )
{
new HashSet<>( ( (Group) store ).getConstituents() ).forEach(
Group grp = (Group) store;
new HashSet<>( grp.getConstituents() ).forEach(
( key ) -> affectedByStores.computeIfAbsent( key, k -> new HashSet<>() )
.remove( store.getKey() ) );

logger.info( "Removed affected-by reverse mapping for: {} in {} member stores", store.getKey(), grp.getConstituents().size() );
}
else
{
Expand Down Expand Up @@ -356,9 +334,50 @@ else if ( action == STORE )
removed.forEach( ( key ) -> affectedByStores.computeIfAbsent( key, k -> new HashSet<>() )
.remove( store.getKey() ) );

logger.info( "Removed affected-by reverse mapping for: {} in {} member stores", store.getKey(), removed.size() );

added.forEach( ( key ) -> affectedByStores.computeIfAbsent( key, k -> new HashSet<>() )
.add( store.getKey() ) );

logger.info( "Added affected-by reverse mapping for: {} in {} member stores", store.getKey(), added.size() );
}
}
}

public void initAffectedBy()
{
final Set<ArtifactStore> allStores = getAllArtifactStores();
allStores.stream().filter( s -> group == s.getType() ).forEach( s -> refreshAffectedBy( s, null, STORE ) );

checkAffectedByCacheHealth();
}

private void checkAffectedByCacheHealth()
{
if ( affectedByStores.isEmpty() )
{
logger.error( "Affected-by reverse mapping appears to have failed. The affected-by cache is empty!" );
}
}

public void initByPkgMap()
{
// re-fill the stores by package cache each time when reboot
if ( storesByPkg != null )
{
logger.info( "Clean the stores-by-pkg cache" );
storesByPkg.clear();
}

final Set<ArtifactStore> allStores = getAllArtifactStores();
logger.info( "There are {} stores need to fill in stores-by-pkg cache", allStores.size() );
for ( ArtifactStore store : allStores )
{
final Map<StoreType, Set<StoreKey>> typedKeys =
storesByPkg.computeIfAbsent( store.getKey().getPackageType(), k -> new HashMap<>() );

final Set<StoreKey> keys = typedKeys.computeIfAbsent( store.getKey().getType(), k -> new HashSet<>() );
keys.add( store.getKey() );
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -26,15 +26,13 @@
import org.commonjava.indy.model.core.io.IndyObjectMapper;
import org.commonjava.indy.subsys.datafile.DataFile;
import org.commonjava.indy.subsys.datafile.DataFileManager;
import org.commonjava.indy.subsys.infinispan.CacheHandle;
import org.commonjava.indy.subsys.infinispan.CacheProducer;
import org.infinispan.manager.EmbeddedCacheManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.inject.Inject;
import javax.inject.Named;

import java.io.IOException;
import java.util.Collections;
import java.util.HashSet;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
package org.commonjava.indy.infinispan.data;

import org.commonjava.indy.action.IndyLifecycleException;
import org.commonjava.indy.action.StartupAction;
import org.commonjava.indy.data.IndyDataException;
import org.commonjava.indy.data.StoreDataManager;

import javax.enterprise.context.ApplicationScoped;
import javax.inject.Inject;

@ApplicationScoped
public class InfinispanStoreDataReverseMapStartupAction
Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

One concern for this startup action: will this be conflict with InfinispanStoreDataManager.init()? I think the init() also did some initialization on the reverse-map.

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

So, this is a great point. It probably indicates that we found a different bug with the new functional test, when we used the migration action as a data-loading mechanism. I've moved both by-pkg and affected-by out to startup actions, so they'll work with migrated data as well as stuff we already have stored.

I'm going to work on a different test setup that doesn't use the migration action to load the initial repo definitions, and see if I can get it to break.

implements StartupAction
{
@Inject
private StoreDataManager storeDataManager;

@Override
public void start()
{
if ( storeDataManager instanceof InfinispanStoreDataManager )
{
( (InfinispanStoreDataManager) storeDataManager ).initAffectedBy();
}
}

@Override
public int getStartupPriority()
{
return 10;
}

@Override
public String getId()
{
return "Infinispan affected-by reverse-map initializer";
}
}
4 changes: 4 additions & 0 deletions ftests/core/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,10 @@
<name>Indy :: Functional Tests :: Core</name>

<dependencies>
<dependency>
<groupId>org.commonjava.indy</groupId>
<artifactId>indy-db-infinispan</artifactId>
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>indy-client-core-java</artifactId>
Expand Down
Loading