diff --git a/api/src/main/java/org/commonjava/indy/data/StoreDataManager.java b/api/src/main/java/org/commonjava/indy/data/StoreDataManager.java index 152d630f9f..c23d61d154 100644 --- a/api/src/main/java/org/commonjava/indy/data/StoreDataManager.java +++ b/api/src/main/java/org/commonjava/indy/data/StoreDataManager.java @@ -134,4 +134,8 @@ void reload() * Stream of StoreKey instances present in the system. */ Stream streamArtifactStoreKeys(); + + Set getStoreKeysByPkg( String pkg ); + + Set getStoreKeysByPkgAndType( final String pkg, final StoreType type ); } diff --git a/db/common/src/main/java/org/commonjava/indy/db/common/AbstractStoreDataManager.java b/db/common/src/main/java/org/commonjava/indy/db/common/AbstractStoreDataManager.java index eff2deda03..06705dd34b 100644 --- a/db/common/src/main/java/org/commonjava/indy/db/common/AbstractStoreDataManager.java +++ b/db/common/src/main/java/org/commonjava/indy/db/common/AbstractStoreDataManager.java @@ -20,7 +20,12 @@ import org.commonjava.indy.change.event.ArtifactStoreUpdateType; import org.commonjava.indy.conf.InternalFeatureConfig; import org.commonjava.indy.conf.SslValidationConfig; -import org.commonjava.indy.data.*; +import org.commonjava.indy.data.ArtifactStoreQuery; +import org.commonjava.indy.data.ArtifactStoreValidateData; +import org.commonjava.indy.data.IndyDataException; +import org.commonjava.indy.data.StoreDataManager; +import org.commonjava.indy.data.StoreEventDispatcher; +import org.commonjava.indy.data.StoreValidator; import org.commonjava.indy.measure.annotation.Measure; import org.commonjava.indy.model.core.ArtifactStore; import org.commonjava.indy.model.core.HostedRepository; @@ -32,7 +37,6 @@ import org.slf4j.LoggerFactory; import javax.inject.Inject; -import java.net.MalformedURLException; import java.util.Collections; import java.util.Map; import java.util.Set; @@ -40,6 +44,7 @@ import java.util.concurrent.locks.ReentrantLock; import java.util.function.BiFunction; import java.util.function.Function; +import java.util.stream.Collectors; import java.util.stream.Stream; import static org.commonjava.indy.model.core.StoreType.hosted; @@ -404,4 +409,17 @@ public void disableNotValidStore(ArtifactStore store,ArtifactStoreValidateData v // } } + @Override + public Set getStoreKeysByPkg( String pkg ) + { + return streamArtifactStoreKeys().filter( key -> key.getPackageType().equals( pkg ) ) + .collect( Collectors.toSet() ); + } + + @Override + public Set getStoreKeysByPkgAndType( final String pkg, final StoreType type ) + { + return streamArtifactStoreKeys().filter( key -> key.getPackageType().equals( pkg ) && key.getType() == type ) + .collect( Collectors.toSet() ); + } } diff --git a/db/common/src/main/java/org/commonjava/indy/db/common/DefaultArtifactStoreQuery.java b/db/common/src/main/java/org/commonjava/indy/db/common/DefaultArtifactStoreQuery.java index 88cbbfd6c9..3426a27e06 100644 --- a/db/common/src/main/java/org/commonjava/indy/db/common/DefaultArtifactStoreQuery.java +++ b/db/common/src/main/java/org/commonjava/indy/db/common/DefaultArtifactStoreQuery.java @@ -15,6 +15,7 @@ */ package org.commonjava.indy.db.common; +import org.apache.commons.lang3.StringUtils; import org.commonjava.indy.data.ArtifactStoreQuery; import org.commonjava.indy.data.IndyDataException; import org.commonjava.indy.data.StoreDataManager; @@ -46,6 +47,8 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import static org.commonjava.indy.model.core.StoreType.group; + /** * This query interface is intended to be reusable across any {@link StoreDataManager} implementation. It contains logic * for working with the {@link ArtifactStore}s contained in the StoreDataManager, but this logic is not tied directly to @@ -122,7 +125,7 @@ else if ( HostedRepository.class.equals( storeCls ) ) } else { - this.types = Collections.singleton( StoreType.group ); + this.types = Collections.singleton( group ); } return (DefaultArtifactStoreQuery) this; @@ -414,8 +417,7 @@ public Set getGroupsAffectedBy( Collection keys ) Set processed = new HashSet<>(); - Set all = new DefaultArtifactStoreQuery<>( dataManager, toProcess.get( 0 ).getPackageType(), null, - Group.class ).keyStream().collect( Collectors.toSet() ); + Set all = dataManager.getStoreKeysByPkgAndType( toProcess.get( 0 ).getPackageType(), group ); logger.debug( "There are {} groups need to loop checking for affected by", all.size() ); @@ -470,12 +472,16 @@ public Stream keyStream() public Stream keyStream( Predicate filterPredicate ) { - return dataManager.streamArtifactStoreKeys().filter(key -> { - if ( packageType != null && !key.getPackageType().equals( packageType )) - { - return false; - } - + final Stream storeKeys; + if ( StringUtils.isNotBlank( this.packageType ) ) + { + storeKeys = dataManager.getStoreKeysByPkg( this.packageType ).stream(); + } + else + { + storeKeys = dataManager.streamArtifactStoreKeys(); + } + return storeKeys.filter(key -> { if ( types != null && !types.isEmpty() && !types.contains( key.getType() ) ) { return false; @@ -553,7 +559,7 @@ public HostedRepository getHostedRepository( final String name ) public Group getGroup( final String name ) throws IndyDataException { - return (Group) dataManager.getArtifactStore( new StoreKey( packageType, StoreType.group, name ) ); + return (Group) dataManager.getArtifactStore( new StoreKey( packageType, group, name ) ); } @Override @@ -572,7 +578,7 @@ private List getGroupOrdering( final String groupName, throw new IndyDataException( "packageType must be set on the query before calling this method!" ); } - final Group master = (Group) dataManager.getArtifactStore( new StoreKey( packageType, StoreType.group, groupName ) ); + final Group master = (Group) dataManager.getArtifactStore( new StoreKey( packageType, group, groupName ) ); if ( master == null ) { return Collections.emptyList(); @@ -617,7 +623,7 @@ private void recurseGroup( final Group master, final StoreType type = key.getType(); try { - if ( recurseGroups && type == StoreType.group ) + if ( recurseGroups && type == group ) { // if we're here, we're definitely recursing groups... Group group = (Group) dataManager.getArtifactStore( key ); diff --git a/db/infinispan/src/main/java/org/commonjava/indy/infinispan/data/InfinispanStoreDataManager.java b/db/infinispan/src/main/java/org/commonjava/indy/infinispan/data/InfinispanStoreDataManager.java index afbe27f5f6..d10bcadae9 100644 --- a/db/infinispan/src/main/java/org/commonjava/indy/infinispan/data/InfinispanStoreDataManager.java +++ b/db/infinispan/src/main/java/org/commonjava/indy/infinispan/data/InfinispanStoreDataManager.java @@ -22,6 +22,7 @@ import org.commonjava.indy.measure.annotation.Measure; import org.commonjava.indy.model.core.ArtifactStore; import org.commonjava.indy.model.core.StoreKey; +import org.commonjava.indy.model.core.StoreType; import org.commonjava.indy.subsys.infinispan.CacheHandle; import org.infinispan.Cache; import org.slf4j.Logger; @@ -31,12 +32,14 @@ import javax.enterprise.context.ApplicationScoped; import javax.enterprise.inject.Alternative; import javax.inject.Inject; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Set; import java.util.stream.Stream; +import static org.commonjava.indy.infinispan.data.StoreDataCacheProducer.STORE_BY_PKG_CACHE; import static org.commonjava.indy.infinispan.data.StoreDataCacheProducer.STORE_DATA_CACHE; @ApplicationScoped @@ -50,6 +53,10 @@ public class InfinispanStoreDataManager @StoreDataCache private CacheHandle stores; + @Inject + @StoreByPkgCache + private CacheHandle>> storesByPkg; + @Inject private StoreEventDispatcher dispatcher; @@ -65,14 +72,31 @@ protected InfinispanStoreDataManager() } @PostConstruct - private void init() + synchronized void init() { + // re-fill the stores by package cache each time when reboot + if ( storesByPkg != null ) + { + logger.info( "Clean the stores-by-pkg cache" ); + storesByPkg.clear(); + } + final Set allStores = getAllArtifactStores(); + logger.info( "There are {} stores need to fill in stores-by-pkg cache", allStores.size() ); + for ( ArtifactStore store : allStores ) + { + final Map> typedKeys = + storesByPkg.computeIfAbsent( store.getKey().getPackageType(), k -> new HashMap<>() ); + final Set keys = typedKeys.computeIfAbsent( store.getKey().getType(), k -> new HashSet<>() ); + keys.add( store.getKey() ); + } } - public InfinispanStoreDataManager( final Cache cache ) + public InfinispanStoreDataManager( final Cache cache, + final Cache>> storesByPkg ) { this.dispatcher = new NoOpStoreEventDispatcher(); this.stores = new CacheHandle( STORE_DATA_CACHE, cache ); + this.storesByPkg = new CacheHandle( STORE_BY_PKG_CACHE, storesByPkg ); } @Override @@ -81,17 +105,27 @@ protected ArtifactStore getArtifactStoreInternal( StoreKey key ) return stores.get( key ); } - @Override - protected ArtifactStore removeArtifactStoreInternal( StoreKey key ) + protected synchronized ArtifactStore removeArtifactStoreInternal( StoreKey key ) { - return stores.remove( key ); + final ArtifactStore store = stores.remove( key ); + final Map> typedKeys = storesByPkg.get( key.getPackageType() ); + if ( typedKeys != null ) + { + final Set keys = typedKeys.get( key.getType() ); + if ( keys != null ) + { + keys.remove( key ); + } + } + return store; } @Override public void clear( final ChangeSummary summary ) { stores.clear(); + storesByPkg.clear(); } @Override @@ -139,9 +173,49 @@ public Stream streamArtifactStoreKeys() } @Override - protected ArtifactStore putArtifactStoreInternal( StoreKey storeKey, ArtifactStore store ) + protected synchronized ArtifactStore putArtifactStoreInternal( StoreKey storeKey, ArtifactStore store ) + { + final ArtifactStore added = stores.put( storeKey, store ); + final Map> typedKeys = + storesByPkg.computeIfAbsent( storeKey.getPackageType(), k -> new HashMap<>() ); + final Set keys = typedKeys.computeIfAbsent( storeKey.getType(), k -> new HashSet<>() ); + keys.add( storeKey ); + return added; + } + + @Override + public Set getStoreKeysByPkg( final String pkg ) + { + final Map> typedKeys = storesByPkg.get( pkg ); + if ( typedKeys != null ) + { + final Set keys = new HashSet<>(); + typedKeys.values().forEach( keys::addAll ); + logger.trace( "There are {} stores for package type {}", keys.size(), pkg ); + return keys; + } + else + { + logger.trace( "There is no store for package type {}", pkg ); + return Collections.emptySet(); + } + } + + @Override + public Set getStoreKeysByPkgAndType( final String pkg, final StoreType type ) { - return stores.put( storeKey, store ); + final Map> typedKeys = storesByPkg.get( pkg ); + if ( typedKeys != null ) + { + final Set keys = typedKeys.get( type ); + if ( keys != null ) + { + logger.trace( "There are {} stores for package type {} with type {}", keys.size(), pkg, type ); + return new HashSet<>( keys ); + } + } + logger.trace( "There is no store for package type {} with type {}", pkg, type ); + return Collections.emptySet(); } } diff --git a/db/infinispan/src/main/java/org/commonjava/indy/infinispan/data/StoreByPkgCache.java b/db/infinispan/src/main/java/org/commonjava/indy/infinispan/data/StoreByPkgCache.java new file mode 100644 index 0000000000..bf973e3ad6 --- /dev/null +++ b/db/infinispan/src/main/java/org/commonjava/indy/infinispan/data/StoreByPkgCache.java @@ -0,0 +1,16 @@ +package org.commonjava.indy.infinispan.data; + +import javax.inject.Qualifier; +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Qualifier +@Target( { ElementType.FIELD, ElementType.PARAMETER, ElementType.METHOD}) +@Retention( RetentionPolicy.RUNTIME) +@Documented +public @interface StoreByPkgCache +{ +} diff --git a/db/infinispan/src/main/java/org/commonjava/indy/infinispan/data/StoreDataCacheProducer.java b/db/infinispan/src/main/java/org/commonjava/indy/infinispan/data/StoreDataCacheProducer.java index e7b4dcedc3..9b8ceb1b67 100644 --- a/db/infinispan/src/main/java/org/commonjava/indy/infinispan/data/StoreDataCacheProducer.java +++ b/db/infinispan/src/main/java/org/commonjava/indy/infinispan/data/StoreDataCacheProducer.java @@ -17,17 +17,22 @@ import org.commonjava.indy.model.core.ArtifactStore; import org.commonjava.indy.model.core.StoreKey; +import org.commonjava.indy.model.core.StoreType; import org.commonjava.indy.subsys.infinispan.CacheHandle; import org.commonjava.indy.subsys.infinispan.CacheProducer; import javax.enterprise.context.ApplicationScoped; import javax.enterprise.inject.Produces; import javax.inject.Inject; +import java.util.Map; +import java.util.Set; public class StoreDataCacheProducer { public static final String STORE_DATA_CACHE = "store-data-v2"; + public static final String STORE_BY_PKG_CACHE = "store-by-package"; + @Inject private CacheProducer cacheProducer; @@ -47,5 +52,12 @@ public CacheHandle getStoreDataCache() // return cacheProducer.getCache( STORE_DATA_CACHE ); // } + @StoreByPkgCache + @Produces + @ApplicationScoped + public CacheHandle>> getStoreByPkgCache() + { + return cacheProducer.getCache( STORE_BY_PKG_CACHE ); + } } \ No newline at end of file diff --git a/db/infinispan/src/test/src/org/commonjava/indy/infinispan/data/InfinispanTCKFixtureProvider.java b/db/infinispan/src/test/src/org/commonjava/indy/infinispan/data/InfinispanTCKFixtureProvider.java index aa1022097a..aeb8da32de 100644 --- a/db/infinispan/src/test/src/org/commonjava/indy/infinispan/data/InfinispanTCKFixtureProvider.java +++ b/db/infinispan/src/test/src/org/commonjava/indy/infinispan/data/InfinispanTCKFixtureProvider.java @@ -19,10 +19,15 @@ import org.commonjava.indy.data.StoreDataManager; import org.commonjava.indy.model.core.ArtifactStore; import org.commonjava.indy.model.core.StoreKey; +import org.commonjava.indy.model.core.StoreType; import org.infinispan.Cache; import org.infinispan.configuration.cache.ConfigurationBuilder; import org.infinispan.manager.DefaultCacheManager; +import java.util.Map; +import java.util.Set; + +import static org.commonjava.indy.infinispan.data.StoreDataCacheProducer.STORE_BY_PKG_CACHE; import static org.commonjava.indy.infinispan.data.StoreDataCacheProducer.STORE_DATA_CACHE; public class InfinispanTCKFixtureProvider @@ -35,7 +40,9 @@ protected void init() DefaultCacheManager cacheManager = new DefaultCacheManager( new ConfigurationBuilder().simpleCache( true ).build() ); Cache storeCache = cacheManager.getCache( STORE_DATA_CACHE, true ); - dataManager = new InfinispanStoreDataManager( storeCache ); + Cache>> storesByPkgCache = cacheManager.getCache( STORE_BY_PKG_CACHE, true ); + dataManager = new InfinispanStoreDataManager( storeCache, storesByPkgCache ); + dataManager.init(); } @Override diff --git a/subsys/infinispan/src/main/resources/infinispan.xml b/subsys/infinispan/src/main/resources/infinispan.xml index 56dcc69637..483dfc45c3 100644 --- a/subsys/infinispan/src/main/resources/infinispan.xml +++ b/subsys/infinispan/src/main/resources/infinispan.xml @@ -118,12 +118,22 @@ - + + + + + + + near-real-time + local-heap + + +