From 5e999d6e824c6b6f938eeb24ec7c49a14ae9039a Mon Sep 17 00:00:00 2001 From: Gang Li Date: Mon, 23 Dec 2019 13:43:36 +0800 Subject: [PATCH] Limit "affected-by" with full scan opeartion in package and type based set Add a new Cache which is > to store all Store keys based on package and type, so for affected group get method, we can avoid the full scan on all repos to get groups, but just simplified to two map get ops. --- .../indy/data/StoreDataManager.java | 4 + .../db/common/AbstractStoreDataManager.java | 22 ++++- .../db/common/DefaultArtifactStoreQuery.java | 30 ++++--- .../data/InfinispanStoreDataManager.java | 88 +++++++++++++++++-- .../indy/infinispan/data/StoreByPkgCache.java | 16 ++++ .../data/StoreDataCacheProducer.java | 12 +++ .../data/InfinispanTCKFixtureProvider.java | 9 +- .../src/main/resources/infinispan.xml | 12 ++- 8 files changed, 170 insertions(+), 23 deletions(-) create mode 100644 db/infinispan/src/main/java/org/commonjava/indy/infinispan/data/StoreByPkgCache.java diff --git a/api/src/main/java/org/commonjava/indy/data/StoreDataManager.java b/api/src/main/java/org/commonjava/indy/data/StoreDataManager.java index 152d630f9f..c23d61d154 100644 --- a/api/src/main/java/org/commonjava/indy/data/StoreDataManager.java +++ b/api/src/main/java/org/commonjava/indy/data/StoreDataManager.java @@ -134,4 +134,8 @@ void reload() * Stream of StoreKey instances present in the system. */ Stream streamArtifactStoreKeys(); + + Set getStoreKeysByPkg( String pkg ); + + Set getStoreKeysByPkgAndType( final String pkg, final StoreType type ); } diff --git a/db/common/src/main/java/org/commonjava/indy/db/common/AbstractStoreDataManager.java b/db/common/src/main/java/org/commonjava/indy/db/common/AbstractStoreDataManager.java index eff2deda03..06705dd34b 100644 --- a/db/common/src/main/java/org/commonjava/indy/db/common/AbstractStoreDataManager.java +++ b/db/common/src/main/java/org/commonjava/indy/db/common/AbstractStoreDataManager.java @@ -20,7 +20,12 @@ import org.commonjava.indy.change.event.ArtifactStoreUpdateType; import org.commonjava.indy.conf.InternalFeatureConfig; import org.commonjava.indy.conf.SslValidationConfig; -import org.commonjava.indy.data.*; +import org.commonjava.indy.data.ArtifactStoreQuery; +import org.commonjava.indy.data.ArtifactStoreValidateData; +import org.commonjava.indy.data.IndyDataException; +import org.commonjava.indy.data.StoreDataManager; +import org.commonjava.indy.data.StoreEventDispatcher; +import org.commonjava.indy.data.StoreValidator; import org.commonjava.indy.measure.annotation.Measure; import org.commonjava.indy.model.core.ArtifactStore; import org.commonjava.indy.model.core.HostedRepository; @@ -32,7 +37,6 @@ import org.slf4j.LoggerFactory; import javax.inject.Inject; -import java.net.MalformedURLException; import java.util.Collections; import java.util.Map; import java.util.Set; @@ -40,6 +44,7 @@ import java.util.concurrent.locks.ReentrantLock; import java.util.function.BiFunction; import java.util.function.Function; +import java.util.stream.Collectors; import java.util.stream.Stream; import static org.commonjava.indy.model.core.StoreType.hosted; @@ -404,4 +409,17 @@ public void disableNotValidStore(ArtifactStore store,ArtifactStoreValidateData v // } } + @Override + public Set getStoreKeysByPkg( String pkg ) + { + return streamArtifactStoreKeys().filter( key -> key.getPackageType().equals( pkg ) ) + .collect( Collectors.toSet() ); + } + + @Override + public Set getStoreKeysByPkgAndType( final String pkg, final StoreType type ) + { + return streamArtifactStoreKeys().filter( key -> key.getPackageType().equals( pkg ) && key.getType() == type ) + .collect( Collectors.toSet() ); + } } diff --git a/db/common/src/main/java/org/commonjava/indy/db/common/DefaultArtifactStoreQuery.java b/db/common/src/main/java/org/commonjava/indy/db/common/DefaultArtifactStoreQuery.java index 88cbbfd6c9..3426a27e06 100644 --- a/db/common/src/main/java/org/commonjava/indy/db/common/DefaultArtifactStoreQuery.java +++ b/db/common/src/main/java/org/commonjava/indy/db/common/DefaultArtifactStoreQuery.java @@ -15,6 +15,7 @@ */ package org.commonjava.indy.db.common; +import org.apache.commons.lang3.StringUtils; import org.commonjava.indy.data.ArtifactStoreQuery; import org.commonjava.indy.data.IndyDataException; import org.commonjava.indy.data.StoreDataManager; @@ -46,6 +47,8 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import static org.commonjava.indy.model.core.StoreType.group; + /** * This query interface is intended to be reusable across any {@link StoreDataManager} implementation. It contains logic * for working with the {@link ArtifactStore}s contained in the StoreDataManager, but this logic is not tied directly to @@ -122,7 +125,7 @@ else if ( HostedRepository.class.equals( storeCls ) ) } else { - this.types = Collections.singleton( StoreType.group ); + this.types = Collections.singleton( group ); } return (DefaultArtifactStoreQuery) this; @@ -414,8 +417,7 @@ public Set getGroupsAffectedBy( Collection keys ) Set processed = new HashSet<>(); - Set all = new DefaultArtifactStoreQuery<>( dataManager, toProcess.get( 0 ).getPackageType(), null, - Group.class ).keyStream().collect( Collectors.toSet() ); + Set all = dataManager.getStoreKeysByPkgAndType( toProcess.get( 0 ).getPackageType(), group ); logger.debug( "There are {} groups need to loop checking for affected by", all.size() ); @@ -470,12 +472,16 @@ public Stream keyStream() public Stream keyStream( Predicate filterPredicate ) { - return dataManager.streamArtifactStoreKeys().filter(key -> { - if ( packageType != null && !key.getPackageType().equals( packageType )) - { - return false; - } - + final Stream storeKeys; + if ( StringUtils.isNotBlank( this.packageType ) ) + { + storeKeys = dataManager.getStoreKeysByPkg( this.packageType ).stream(); + } + else + { + storeKeys = dataManager.streamArtifactStoreKeys(); + } + return storeKeys.filter(key -> { if ( types != null && !types.isEmpty() && !types.contains( key.getType() ) ) { return false; @@ -553,7 +559,7 @@ public HostedRepository getHostedRepository( final String name ) public Group getGroup( final String name ) throws IndyDataException { - return (Group) dataManager.getArtifactStore( new StoreKey( packageType, StoreType.group, name ) ); + return (Group) dataManager.getArtifactStore( new StoreKey( packageType, group, name ) ); } @Override @@ -572,7 +578,7 @@ private List getGroupOrdering( final String groupName, throw new IndyDataException( "packageType must be set on the query before calling this method!" ); } - final Group master = (Group) dataManager.getArtifactStore( new StoreKey( packageType, StoreType.group, groupName ) ); + final Group master = (Group) dataManager.getArtifactStore( new StoreKey( packageType, group, groupName ) ); if ( master == null ) { return Collections.emptyList(); @@ -617,7 +623,7 @@ private void recurseGroup( final Group master, final StoreType type = key.getType(); try { - if ( recurseGroups && type == StoreType.group ) + if ( recurseGroups && type == group ) { // if we're here, we're definitely recursing groups... Group group = (Group) dataManager.getArtifactStore( key ); diff --git a/db/infinispan/src/main/java/org/commonjava/indy/infinispan/data/InfinispanStoreDataManager.java b/db/infinispan/src/main/java/org/commonjava/indy/infinispan/data/InfinispanStoreDataManager.java index afbe27f5f6..d10bcadae9 100644 --- a/db/infinispan/src/main/java/org/commonjava/indy/infinispan/data/InfinispanStoreDataManager.java +++ b/db/infinispan/src/main/java/org/commonjava/indy/infinispan/data/InfinispanStoreDataManager.java @@ -22,6 +22,7 @@ import org.commonjava.indy.measure.annotation.Measure; import org.commonjava.indy.model.core.ArtifactStore; import org.commonjava.indy.model.core.StoreKey; +import org.commonjava.indy.model.core.StoreType; import org.commonjava.indy.subsys.infinispan.CacheHandle; import org.infinispan.Cache; import org.slf4j.Logger; @@ -31,12 +32,14 @@ import javax.enterprise.context.ApplicationScoped; import javax.enterprise.inject.Alternative; import javax.inject.Inject; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Set; import java.util.stream.Stream; +import static org.commonjava.indy.infinispan.data.StoreDataCacheProducer.STORE_BY_PKG_CACHE; import static org.commonjava.indy.infinispan.data.StoreDataCacheProducer.STORE_DATA_CACHE; @ApplicationScoped @@ -50,6 +53,10 @@ public class InfinispanStoreDataManager @StoreDataCache private CacheHandle stores; + @Inject + @StoreByPkgCache + private CacheHandle>> storesByPkg; + @Inject private StoreEventDispatcher dispatcher; @@ -65,14 +72,31 @@ protected InfinispanStoreDataManager() } @PostConstruct - private void init() + synchronized void init() { + // re-fill the stores by package cache each time when reboot + if ( storesByPkg != null ) + { + logger.info( "Clean the stores-by-pkg cache" ); + storesByPkg.clear(); + } + final Set allStores = getAllArtifactStores(); + logger.info( "There are {} stores need to fill in stores-by-pkg cache", allStores.size() ); + for ( ArtifactStore store : allStores ) + { + final Map> typedKeys = + storesByPkg.computeIfAbsent( store.getKey().getPackageType(), k -> new HashMap<>() ); + final Set keys = typedKeys.computeIfAbsent( store.getKey().getType(), k -> new HashSet<>() ); + keys.add( store.getKey() ); + } } - public InfinispanStoreDataManager( final Cache cache ) + public InfinispanStoreDataManager( final Cache cache, + final Cache>> storesByPkg ) { this.dispatcher = new NoOpStoreEventDispatcher(); this.stores = new CacheHandle( STORE_DATA_CACHE, cache ); + this.storesByPkg = new CacheHandle( STORE_BY_PKG_CACHE, storesByPkg ); } @Override @@ -81,17 +105,27 @@ protected ArtifactStore getArtifactStoreInternal( StoreKey key ) return stores.get( key ); } - @Override - protected ArtifactStore removeArtifactStoreInternal( StoreKey key ) + protected synchronized ArtifactStore removeArtifactStoreInternal( StoreKey key ) { - return stores.remove( key ); + final ArtifactStore store = stores.remove( key ); + final Map> typedKeys = storesByPkg.get( key.getPackageType() ); + if ( typedKeys != null ) + { + final Set keys = typedKeys.get( key.getType() ); + if ( keys != null ) + { + keys.remove( key ); + } + } + return store; } @Override public void clear( final ChangeSummary summary ) { stores.clear(); + storesByPkg.clear(); } @Override @@ -139,9 +173,49 @@ public Stream streamArtifactStoreKeys() } @Override - protected ArtifactStore putArtifactStoreInternal( StoreKey storeKey, ArtifactStore store ) + protected synchronized ArtifactStore putArtifactStoreInternal( StoreKey storeKey, ArtifactStore store ) + { + final ArtifactStore added = stores.put( storeKey, store ); + final Map> typedKeys = + storesByPkg.computeIfAbsent( storeKey.getPackageType(), k -> new HashMap<>() ); + final Set keys = typedKeys.computeIfAbsent( storeKey.getType(), k -> new HashSet<>() ); + keys.add( storeKey ); + return added; + } + + @Override + public Set getStoreKeysByPkg( final String pkg ) + { + final Map> typedKeys = storesByPkg.get( pkg ); + if ( typedKeys != null ) + { + final Set keys = new HashSet<>(); + typedKeys.values().forEach( keys::addAll ); + logger.trace( "There are {} stores for package type {}", keys.size(), pkg ); + return keys; + } + else + { + logger.trace( "There is no store for package type {}", pkg ); + return Collections.emptySet(); + } + } + + @Override + public Set getStoreKeysByPkgAndType( final String pkg, final StoreType type ) { - return stores.put( storeKey, store ); + final Map> typedKeys = storesByPkg.get( pkg ); + if ( typedKeys != null ) + { + final Set keys = typedKeys.get( type ); + if ( keys != null ) + { + logger.trace( "There are {} stores for package type {} with type {}", keys.size(), pkg, type ); + return new HashSet<>( keys ); + } + } + logger.trace( "There is no store for package type {} with type {}", pkg, type ); + return Collections.emptySet(); } } diff --git a/db/infinispan/src/main/java/org/commonjava/indy/infinispan/data/StoreByPkgCache.java b/db/infinispan/src/main/java/org/commonjava/indy/infinispan/data/StoreByPkgCache.java new file mode 100644 index 0000000000..bf973e3ad6 --- /dev/null +++ b/db/infinispan/src/main/java/org/commonjava/indy/infinispan/data/StoreByPkgCache.java @@ -0,0 +1,16 @@ +package org.commonjava.indy.infinispan.data; + +import javax.inject.Qualifier; +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Qualifier +@Target( { ElementType.FIELD, ElementType.PARAMETER, ElementType.METHOD}) +@Retention( RetentionPolicy.RUNTIME) +@Documented +public @interface StoreByPkgCache +{ +} diff --git a/db/infinispan/src/main/java/org/commonjava/indy/infinispan/data/StoreDataCacheProducer.java b/db/infinispan/src/main/java/org/commonjava/indy/infinispan/data/StoreDataCacheProducer.java index e7b4dcedc3..9b8ceb1b67 100644 --- a/db/infinispan/src/main/java/org/commonjava/indy/infinispan/data/StoreDataCacheProducer.java +++ b/db/infinispan/src/main/java/org/commonjava/indy/infinispan/data/StoreDataCacheProducer.java @@ -17,17 +17,22 @@ import org.commonjava.indy.model.core.ArtifactStore; import org.commonjava.indy.model.core.StoreKey; +import org.commonjava.indy.model.core.StoreType; import org.commonjava.indy.subsys.infinispan.CacheHandle; import org.commonjava.indy.subsys.infinispan.CacheProducer; import javax.enterprise.context.ApplicationScoped; import javax.enterprise.inject.Produces; import javax.inject.Inject; +import java.util.Map; +import java.util.Set; public class StoreDataCacheProducer { public static final String STORE_DATA_CACHE = "store-data-v2"; + public static final String STORE_BY_PKG_CACHE = "store-by-package"; + @Inject private CacheProducer cacheProducer; @@ -47,5 +52,12 @@ public CacheHandle getStoreDataCache() // return cacheProducer.getCache( STORE_DATA_CACHE ); // } + @StoreByPkgCache + @Produces + @ApplicationScoped + public CacheHandle>> getStoreByPkgCache() + { + return cacheProducer.getCache( STORE_BY_PKG_CACHE ); + } } \ No newline at end of file diff --git a/db/infinispan/src/test/src/org/commonjava/indy/infinispan/data/InfinispanTCKFixtureProvider.java b/db/infinispan/src/test/src/org/commonjava/indy/infinispan/data/InfinispanTCKFixtureProvider.java index aa1022097a..aeb8da32de 100644 --- a/db/infinispan/src/test/src/org/commonjava/indy/infinispan/data/InfinispanTCKFixtureProvider.java +++ b/db/infinispan/src/test/src/org/commonjava/indy/infinispan/data/InfinispanTCKFixtureProvider.java @@ -19,10 +19,15 @@ import org.commonjava.indy.data.StoreDataManager; import org.commonjava.indy.model.core.ArtifactStore; import org.commonjava.indy.model.core.StoreKey; +import org.commonjava.indy.model.core.StoreType; import org.infinispan.Cache; import org.infinispan.configuration.cache.ConfigurationBuilder; import org.infinispan.manager.DefaultCacheManager; +import java.util.Map; +import java.util.Set; + +import static org.commonjava.indy.infinispan.data.StoreDataCacheProducer.STORE_BY_PKG_CACHE; import static org.commonjava.indy.infinispan.data.StoreDataCacheProducer.STORE_DATA_CACHE; public class InfinispanTCKFixtureProvider @@ -35,7 +40,9 @@ protected void init() DefaultCacheManager cacheManager = new DefaultCacheManager( new ConfigurationBuilder().simpleCache( true ).build() ); Cache storeCache = cacheManager.getCache( STORE_DATA_CACHE, true ); - dataManager = new InfinispanStoreDataManager( storeCache ); + Cache>> storesByPkgCache = cacheManager.getCache( STORE_BY_PKG_CACHE, true ); + dataManager = new InfinispanStoreDataManager( storeCache, storesByPkgCache ); + dataManager.init(); } @Override diff --git a/subsys/infinispan/src/main/resources/infinispan.xml b/subsys/infinispan/src/main/resources/infinispan.xml index 56dcc69637..483dfc45c3 100644 --- a/subsys/infinispan/src/main/resources/infinispan.xml +++ b/subsys/infinispan/src/main/resources/infinispan.xml @@ -118,12 +118,22 @@ - + + + + + + + near-real-time + local-heap + + +