From 6a7bc11a6a9d8fade783b9442b7aaa4aa4d3f915 Mon Sep 17 00:00:00 2001 From: John Casey Date: Wed, 15 Jan 2020 15:53:56 +0100 Subject: [PATCH 1/4] Fix affected-by reversemap initialization in lifecycle startup --- .../data/InfinispanStoreDataManager.java | 7 + .../InfinispanStoreDataMigrationAction.java | 2 - ...ispanStoreDataReverseMapStartupAction.java | 44 +++ .../content/StoreReverseMapMigrationTest.java | 258 ++++++++++++++++++ 4 files changed, 309 insertions(+), 2 deletions(-) create mode 100644 db/infinispan/src/main/java/org/commonjava/indy/infinispan/data/InfinispanStoreDataReverseMapStartupAction.java create mode 100644 ftests/core/src/main/java/org/commonjava/indy/ftest/core/content/StoreReverseMapMigrationTest.java diff --git a/db/infinispan/src/main/java/org/commonjava/indy/infinispan/data/InfinispanStoreDataManager.java b/db/infinispan/src/main/java/org/commonjava/indy/infinispan/data/InfinispanStoreDataManager.java index 2dc3a6359a..f8047773bc 100644 --- a/db/infinispan/src/main/java/org/commonjava/indy/infinispan/data/InfinispanStoreDataManager.java +++ b/db/infinispan/src/main/java/org/commonjava/indy/infinispan/data/InfinispanStoreDataManager.java @@ -16,6 +16,7 @@ package org.commonjava.indy.infinispan.data; import org.commonjava.indy.audit.ChangeSummary; +import org.commonjava.indy.data.IndyDataException; import org.commonjava.indy.data.NoOpStoreEventDispatcher; import org.commonjava.indy.data.StoreEventDispatcher; import org.commonjava.indy.db.common.AbstractStoreDataManager; @@ -361,4 +362,10 @@ else if ( action == STORE ) } } } + + public void initAffectedBy() + throws IndyDataException + { + streamArtifactStores().filter( s -> group == s.getType() ).forEach( s -> refreshAffectedBy( s, null, STORE ) ); + } } diff --git a/db/infinispan/src/main/java/org/commonjava/indy/infinispan/data/InfinispanStoreDataMigrationAction.java b/db/infinispan/src/main/java/org/commonjava/indy/infinispan/data/InfinispanStoreDataMigrationAction.java index 0769eef1f0..c2401fb4e1 100644 --- a/db/infinispan/src/main/java/org/commonjava/indy/infinispan/data/InfinispanStoreDataMigrationAction.java +++ b/db/infinispan/src/main/java/org/commonjava/indy/infinispan/data/InfinispanStoreDataMigrationAction.java @@ -26,7 +26,6 @@ import org.commonjava.indy.model.core.io.IndyObjectMapper; import org.commonjava.indy.subsys.datafile.DataFile; import org.commonjava.indy.subsys.datafile.DataFileManager; -import org.commonjava.indy.subsys.infinispan.CacheHandle; import org.commonjava.indy.subsys.infinispan.CacheProducer; import org.infinispan.manager.EmbeddedCacheManager; import org.slf4j.Logger; @@ -34,7 +33,6 @@ import javax.inject.Inject; import javax.inject.Named; - import java.io.IOException; import java.util.Collections; import java.util.HashSet; diff --git a/db/infinispan/src/main/java/org/commonjava/indy/infinispan/data/InfinispanStoreDataReverseMapStartupAction.java b/db/infinispan/src/main/java/org/commonjava/indy/infinispan/data/InfinispanStoreDataReverseMapStartupAction.java new file mode 100644 index 0000000000..6356fba3f0 --- /dev/null +++ b/db/infinispan/src/main/java/org/commonjava/indy/infinispan/data/InfinispanStoreDataReverseMapStartupAction.java @@ -0,0 +1,44 @@ +package org.commonjava.indy.infinispan.data; + +import org.commonjava.indy.action.IndyLifecycleException; +import org.commonjava.indy.action.StartupAction; +import org.commonjava.indy.data.IndyDataException; +import org.commonjava.indy.data.StoreDataManager; + +import javax.enterprise.context.ApplicationScoped; +import javax.inject.Inject; + +@ApplicationScoped +public class InfinispanStoreDataReverseMapStartupAction + implements StartupAction +{ + @Inject + private StoreDataManager storeDataManager; + + @Override + public void start() + throws IndyLifecycleException + { + try + { + ((InfinispanStoreDataManager)storeDataManager).initAffectedBy(); + } + catch ( IndyDataException e ) + { + throw new IndyLifecycleException( "Failed to reverse-map stores and groups they affect: " + e.getMessage(), + e ); + } + } + + @Override + public int getStartupPriority() + { + return 10; + } + + @Override + public String getId() + { + return "Infinispan affected-by reverse-map initializer"; + } +} diff --git a/ftests/core/src/main/java/org/commonjava/indy/ftest/core/content/StoreReverseMapMigrationTest.java b/ftests/core/src/main/java/org/commonjava/indy/ftest/core/content/StoreReverseMapMigrationTest.java new file mode 100644 index 0000000000..e86dc250e4 --- /dev/null +++ b/ftests/core/src/main/java/org/commonjava/indy/ftest/core/content/StoreReverseMapMigrationTest.java @@ -0,0 +1,258 @@ +/** + * Copyright (C) 2011-2020 Red Hat, Inc. (https://github.com/Commonjava/indy) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.commonjava.indy.ftest.core.content; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang.StringUtils; +import org.commonjava.indy.client.core.IndyClientException; +import org.commonjava.indy.ftest.core.AbstractContentManagementTest; +import org.commonjava.indy.model.core.Group; +import org.commonjava.indy.model.core.HostedRepository; +import org.commonjava.indy.model.core.StoreType; +import org.commonjava.indy.model.core.io.IndyObjectMapper; +import org.commonjava.indy.test.fixture.core.CoreServerFixture; +import org.custommonkey.xmlunit.XMLUnit; +import org.junit.Test; +import org.xml.sax.SAXException; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; + +import static org.commonjava.indy.model.core.StoreType.group; +import static org.commonjava.indy.pkg.PackageTypeConstants.PKG_TYPE_MAVEN; +import static org.custommonkey.xmlunit.XMLAssert.assertXMLEqual; +import static org.junit.Assert.fail; + +/** + * GIVEN: + * + * + * WHEN: + * + * + * THEN: + * + * + */ +public class StoreReverseMapMigrationTest + extends AbstractContentManagementTest +{ + final String repoA = "hostedA"; + final String repoB = "hostedB"; + final String repoC = "hostedC"; + + final String groupX = "groupX"; + final String groupY = "groupY"; + + final String path = "org/foo/bar/maven-metadata.xml"; + + HostedRepository hostedA = new HostedRepository( PKG_TYPE_MAVEN, repoA ); + HostedRepository hostedB = new HostedRepository( PKG_TYPE_MAVEN, repoB ); + HostedRepository hostedC = new HostedRepository( PKG_TYPE_MAVEN, repoC ); + + Group gX = new Group( PKG_TYPE_MAVEN, groupX, hostedA.getKey(), hostedB.getKey() ); + Group gY = new Group( PKG_TYPE_MAVEN,groupY, gX.getKey() ); + + @Override + protected void initTestData( CoreServerFixture fixture ) + throws IOException + { + ObjectMapper mapper = new IndyObjectMapper( true ); + + writeDataFile( "indy/maven/hosted/" + repoA + ".json", mapper.writeValueAsString( hostedA ) ); + writeDataFile( "indy/maven/hosted/" + repoB + ".json", mapper.writeValueAsString( hostedB ) ); + writeDataFile( "indy/maven/hosted/" + repoC + ".json", mapper.writeValueAsString( hostedC ) ); + + writeDataFile( "indy/maven/group/" + groupX + ".json", mapper.writeValueAsString( gX ) ); + writeDataFile( "indy/maven/group/" + groupY + ".json", mapper.writeValueAsString( gY ) ); + + super.initTestData( fixture ); + } + + @Test + public void run() + throws Exception + { + /* @formatter:off */ + final String repoAContent = "\n" + + "\n" + + " org.foo\n" + + " bar\n" + + " \n" + + " 1.0\n" + + " 1.0\n" + + " \n" + + " 1.0\n" + + " \n" + + " 20150721164334\n" + + " \n" + + "\n"; + /* @formatter:on */ + + /* @formatter:off */ + final String repoBContent = "\n" + + "\n" + + " org.foo\n" + + " bar\n" + + " \n" + + " 2.0\n" + + " 2.0\n" + + " \n" + + " 2.0\n" + + " \n" + + " 20160722164334\n" + + " \n" + + "\n"; + /* @formatter:on */ + + /* @formatter:off */ + final String repoCContent = "\n" + + "\n" + + " org.foo\n" + + " bar\n" + + " \n" + + " 3.0\n" + + " 3.0\n" + + " \n" + + " 3.0\n" + + " \n" + + " 20160723164334\n" + + " \n" + + "\n"; + /* @formatter:on */ + + createHostedStorePath( hostedA, path, repoAContent ); + try (final InputStream stream = client.content().get( hostedA.getKey(), path )) + { + assertContent( repoAContent, IOUtils.toString( stream ) ); + } + + createHostedStorePath( hostedB, path, repoBContent ); + try (final InputStream stream = client.content().get( hostedB.getKey(), path )) + { + assertContent( repoBContent, IOUtils.toString( stream ) ); + } + + createHostedStorePath( hostedC, path, repoCContent ); + try (final InputStream stream = client.content().get( hostedC.getKey(), path )) + { + assertContent( repoCContent, IOUtils.toString( stream ) ); + } + + /* @formatter:off */ + final String mergedContent1 = + "\n" + + " \n" + + " org.foo\n" + + " bar\n" + + " \n" + + " 2.0\n" + + " 2.0\n" + + " \n" + + " 1.0\n" + + " 2.0\n" + + " \n" + + " 20160722164334\n" + + " \n" + + "\n"; + /* @formatter:on */ + try (final InputStream stream = client.content().get( group, gY.getName(), path )) + { + assertContent( mergedContent1, IOUtils.toString( stream ) ); + } + + gX.addConstituent( hostedC ); + client.stores().update( gX, "test update" ); + gX = client.stores().load( StoreType.group, gX.getName(), Group.class ); + logger.debug( "\n\nGroup constituents are:\n {}\n\n", StringUtils.join( gX.getConstituents(), "\n " ) ); + + waitForEventPropagation(); + + /* @formatter:off */ + final String mergedContent2 = + "\n" + + " \n" + + " org.foo\n" + + " bar\n" + + " \n" + + " 3.0\n" + + " 3.0\n" + + " \n" + + " 1.0\n" + + " 2.0\n" + + " 3.0\n" + + " \n" + + " 20160723164334\n" + + " \n" + + "\n"; + /* @formatter:on */ + + try (final InputStream stream = client.content().get( group, gY.getName(), path )) + { + assertContent( mergedContent2, IOUtils.toString( stream ) ); + } + } + + @Override + protected boolean createStandardTestStructures() + { + return false; + } + + private HostedRepository createHostedStorePath( final HostedRepository hosted, final String path, final String content ) + throws Exception + { + if(StringUtils.isNotBlank( content )) + { + client.content().store( hosted.getKey(), path, new ByteArrayInputStream( content.getBytes() ) ); + } + return hosted; + } + + private void assertContent( String expectedXml, String actual ) + throws IndyClientException, IOException + { + + logger.debug( "Comparing downloaded XML:\n\n{}\n\nTo expected XML:\n\n{}\n\n", actual, expectedXml ); + + try + { + XMLUnit.setIgnoreWhitespace( true ); + XMLUnit.setIgnoreDiffBetweenTextAndCDATA( true ); + XMLUnit.setIgnoreAttributeOrder( true ); + XMLUnit.setIgnoreComments( true ); + + assertXMLEqual( expectedXml, actual ); + } + catch ( SAXException e ) + { + e.printStackTrace(); + fail( "Downloaded XML not equal to expected XML" ); + } + } +} From df12b219ff142beb3683120c7b8b924cb603a30d Mon Sep 17 00:00:00 2001 From: John Casey Date: Thu, 16 Jan 2020 12:14:15 +0100 Subject: [PATCH 2/4] Add logging to note when the affected-by cache is altered, and another error log message if it's empty --- .../data/InfinispanStoreDataManager.java | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/db/infinispan/src/main/java/org/commonjava/indy/infinispan/data/InfinispanStoreDataManager.java b/db/infinispan/src/main/java/org/commonjava/indy/infinispan/data/InfinispanStoreDataManager.java index f8047773bc..5fe6cd47eb 100644 --- a/db/infinispan/src/main/java/org/commonjava/indy/infinispan/data/InfinispanStoreDataManager.java +++ b/db/infinispan/src/main/java/org/commonjava/indy/infinispan/data/InfinispanStoreDataManager.java @@ -252,6 +252,8 @@ public Set getStoreKeysByPkgAndType( final String pkg, final StoreType @Override public Set affectedBy( final Collection keys ) { + checkAffectedByCacheHealth(); + final Set result = new HashSet<>(); // use these to avoid recursion @@ -311,9 +313,12 @@ protected void refreshAffectedBy( final ArtifactStore store, final ArtifactStore { if ( store instanceof Group ) { - new HashSet<>( ( (Group) store ).getConstituents() ).forEach( + Group grp = (Group) store; + new HashSet<>( grp.getConstituents() ).forEach( ( key ) -> affectedByStores.computeIfAbsent( key, k -> new HashSet<>() ) .remove( store.getKey() ) ); + + logger.info( "Removed affected-by reverse mapping for: {} in {} member stores", store.getKey(), grp.getConstituents().size() ); } else { @@ -357,8 +362,12 @@ else if ( action == STORE ) removed.forEach( ( key ) -> affectedByStores.computeIfAbsent( key, k -> new HashSet<>() ) .remove( store.getKey() ) ); + logger.info( "Removed affected-by reverse mapping for: {} in {} member stores", store.getKey(), removed.size() ); + added.forEach( ( key ) -> affectedByStores.computeIfAbsent( key, k -> new HashSet<>() ) .add( store.getKey() ) ); + + logger.info( "Added affected-by reverse mapping for: {} in {} member stores", store.getKey(), added.size() ); } } } @@ -367,5 +376,15 @@ public void initAffectedBy() throws IndyDataException { streamArtifactStores().filter( s -> group == s.getType() ).forEach( s -> refreshAffectedBy( s, null, STORE ) ); + + checkAffectedByCacheHealth(); + } + + private void checkAffectedByCacheHealth() + { + if ( affectedByStores.isEmpty() ) + { + logger.error( "Affected-by reverse mapping appears to have failed. The affected-by cache is empty!" ); + } } } From 7b6d08df2bc99adac529d1845ac51f3ce5f57bea Mon Sep 17 00:00:00 2001 From: John Casey Date: Thu, 16 Jan 2020 14:50:46 +0100 Subject: [PATCH 3/4] Refactor InfinispanStoreDataManager.init() to startup actions, add some logging The init() method runs before any migration actions run, which means migrated stores won't be stored in the byPkg cache. The old initializer for the affected-by map has a similar problem, so I removed it in favor of the new startup action. The byPkg init was moved out to a startup action too for this reason. Finally, I've added a log message to show which groups were found to be affected by a specific set of input keys. This will give us better information about whether the affected-by map is being used or not. --- .../db/common/AbstractStoreDataManager.java | 1 + ...inispanStoreDataByPkgMapStartupAction.java | 45 +++++++++++++++ .../data/InfinispanStoreDataManager.java | 57 ++++++++----------- ...ispanStoreDataReverseMapStartupAction.java | 17 +++--- 4 files changed, 81 insertions(+), 39 deletions(-) create mode 100644 db/infinispan/src/main/java/org/commonjava/indy/infinispan/data/InfinispanStoreDataByPkgMapStartupAction.java diff --git a/db/common/src/main/java/org/commonjava/indy/db/common/AbstractStoreDataManager.java b/db/common/src/main/java/org/commonjava/indy/db/common/AbstractStoreDataManager.java index a4412f89a0..9c06afd062 100644 --- a/db/common/src/main/java/org/commonjava/indy/db/common/AbstractStoreDataManager.java +++ b/db/common/src/main/java/org/commonjava/indy/db/common/AbstractStoreDataManager.java @@ -511,4 +511,5 @@ protected Set affectedByFromStores( final Collection keys ) return groups; } + } diff --git a/db/infinispan/src/main/java/org/commonjava/indy/infinispan/data/InfinispanStoreDataByPkgMapStartupAction.java b/db/infinispan/src/main/java/org/commonjava/indy/infinispan/data/InfinispanStoreDataByPkgMapStartupAction.java new file mode 100644 index 0000000000..0d780df8bf --- /dev/null +++ b/db/infinispan/src/main/java/org/commonjava/indy/infinispan/data/InfinispanStoreDataByPkgMapStartupAction.java @@ -0,0 +1,45 @@ +package org.commonjava.indy.infinispan.data; + +import org.commonjava.indy.action.IndyLifecycleException; +import org.commonjava.indy.action.StartupAction; +import org.commonjava.indy.data.StoreDataManager; +import org.commonjava.indy.model.core.ArtifactStore; +import org.commonjava.indy.model.core.StoreKey; +import org.commonjava.indy.model.core.StoreType; +import org.commonjava.indy.subsys.infinispan.CacheHandle; + +import javax.inject.Inject; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +public class InfinispanStoreDataByPkgMapStartupAction + implements StartupAction +{ + + @Inject + private StoreDataManager storeDataManager; + + @Override + public void start() + throws IndyLifecycleException + { + if ( storeDataManager instanceof InfinispanStoreDataManager ) + { + ((InfinispanStoreDataManager)storeDataManager).initByPkgMap(); + } + } + + @Override + public int getStartupPriority() + { + return 11; + } + + @Override + public String getId() + { + return "Infinispan by-pkg map initializer"; + } +} diff --git a/db/infinispan/src/main/java/org/commonjava/indy/infinispan/data/InfinispanStoreDataManager.java b/db/infinispan/src/main/java/org/commonjava/indy/infinispan/data/InfinispanStoreDataManager.java index 5fe6cd47eb..b3c6bc6cc7 100644 --- a/db/infinispan/src/main/java/org/commonjava/indy/infinispan/data/InfinispanStoreDataManager.java +++ b/db/infinispan/src/main/java/org/commonjava/indy/infinispan/data/InfinispanStoreDataManager.java @@ -85,37 +85,6 @@ protected InfinispanStoreDataManager() { } - @PostConstruct - synchronized void init() - { - // re-fill the stores by package cache each time when reboot - if ( storesByPkg != null ) - { - logger.info( "Clean the stores-by-pkg cache" ); - storesByPkg.clear(); - } - - boolean affectedByIsEmpty = affectedByStores.isEmpty(); - - final Set allStores = getAllArtifactStores(); - logger.info( "There are {} stores need to fill in stores-by-pkg cache", allStores.size() ); - for ( ArtifactStore store : allStores ) - { - final Map> typedKeys = - storesByPkg.computeIfAbsent( store.getKey().getPackageType(), k -> new HashMap<>() ); - - final Set keys = typedKeys.computeIfAbsent( store.getKey().getType(), k -> new HashSet<>() ); - keys.add( store.getKey() ); - - // If the affected-by cache is empty AND this is a group, record the reverse-map of constituents. - if ( affectedByIsEmpty && store.getType() == group ) - { - new HashSet<>( ( (Group) store ).getConstituents() ).forEach( - key -> affectedByStores.computeIfAbsent( key, k -> new HashSet<>() ).add( store.getKey() ) ); - } - } - } - public InfinispanStoreDataManager( final Cache cache, final Cache>> storesByPkg, final Cache> affectedByStoresCache ) @@ -124,7 +93,7 @@ public InfinispanStoreDataManager( final Cache cache, this.stores = new CacheHandle( STORE_DATA_CACHE, cache ); this.storesByPkg = new CacheHandle( STORE_BY_PKG_CACHE, storesByPkg ); this.affectedByStores = new CacheHandle( AFFECTED_BY_STORE_CACHE, affectedByStoresCache ); - init(); + logger.warn( "Constructor init: STARTUP ACTIONS MAY NOT RUN." ); } @Override @@ -298,6 +267,9 @@ public Set affectedBy( final Collection keys ) } } + logger.info( "Groups affected by {} are: {}", keys, + result.stream().map( ArtifactStore::getKey ).collect( Collectors.toSet() ) ); + return result; } @@ -387,4 +359,25 @@ private void checkAffectedByCacheHealth() logger.error( "Affected-by reverse mapping appears to have failed. The affected-by cache is empty!" ); } } + + public void initByPkgMap() + { + // re-fill the stores by package cache each time when reboot + if ( storesByPkg != null ) + { + logger.info( "Clean the stores-by-pkg cache" ); + storesByPkg.clear(); + } + + final Set allStores = getAllArtifactStores(); + logger.info( "There are {} stores need to fill in stores-by-pkg cache", allStores.size() ); + for ( ArtifactStore store : allStores ) + { + final Map> typedKeys = + storesByPkg.computeIfAbsent( store.getKey().getPackageType(), k -> new HashMap<>() ); + + final Set keys = typedKeys.computeIfAbsent( store.getKey().getType(), k -> new HashSet<>() ); + keys.add( store.getKey() ); + } + } } diff --git a/db/infinispan/src/main/java/org/commonjava/indy/infinispan/data/InfinispanStoreDataReverseMapStartupAction.java b/db/infinispan/src/main/java/org/commonjava/indy/infinispan/data/InfinispanStoreDataReverseMapStartupAction.java index 6356fba3f0..0bdd93b865 100644 --- a/db/infinispan/src/main/java/org/commonjava/indy/infinispan/data/InfinispanStoreDataReverseMapStartupAction.java +++ b/db/infinispan/src/main/java/org/commonjava/indy/infinispan/data/InfinispanStoreDataReverseMapStartupAction.java @@ -19,14 +19,17 @@ public class InfinispanStoreDataReverseMapStartupAction public void start() throws IndyLifecycleException { - try + if ( storeDataManager instanceof InfinispanStoreDataManager ) { - ((InfinispanStoreDataManager)storeDataManager).initAffectedBy(); - } - catch ( IndyDataException e ) - { - throw new IndyLifecycleException( "Failed to reverse-map stores and groups they affect: " + e.getMessage(), - e ); + try + { + ( (InfinispanStoreDataManager) storeDataManager ).initAffectedBy(); + } + catch ( IndyDataException e ) + { + throw new IndyLifecycleException( "Failed to reverse-map stores and groups they affect: " + e.getMessage(), + e ); + } } } From 869e9a67e2c79cf2790ef663ba9c34fa7e3944a4 Mon Sep 17 00:00:00 2001 From: John Casey Date: Thu, 16 Jan 2020 16:53:11 +0100 Subject: [PATCH 4/4] adding another ftest that initializes store data at boot instead of at migration --- ...inispanStoreDataByPkgMapStartupAction.java | 8 - .../data/InfinispanStoreDataManager.java | 4 +- ...ispanStoreDataReverseMapStartupAction.java | 11 +- ftests/core/pom.xml | 4 + .../core/content/StoreReverseMapInitTest.java | 253 ++++++++++++++++++ .../fixture/StoreTestDataBootupAction.java | 83 ++++++ .../core/fixture/StoreTestDataConstants.java | 13 + 7 files changed, 356 insertions(+), 20 deletions(-) create mode 100644 ftests/core/src/main/java/org/commonjava/indy/ftest/core/content/StoreReverseMapInitTest.java create mode 100644 ftests/core/src/main/java/org/commonjava/indy/ftest/core/fixture/StoreTestDataBootupAction.java create mode 100644 ftests/core/src/main/java/org/commonjava/indy/ftest/core/fixture/StoreTestDataConstants.java diff --git a/db/infinispan/src/main/java/org/commonjava/indy/infinispan/data/InfinispanStoreDataByPkgMapStartupAction.java b/db/infinispan/src/main/java/org/commonjava/indy/infinispan/data/InfinispanStoreDataByPkgMapStartupAction.java index 0d780df8bf..bb02757c61 100644 --- a/db/infinispan/src/main/java/org/commonjava/indy/infinispan/data/InfinispanStoreDataByPkgMapStartupAction.java +++ b/db/infinispan/src/main/java/org/commonjava/indy/infinispan/data/InfinispanStoreDataByPkgMapStartupAction.java @@ -3,16 +3,8 @@ import org.commonjava.indy.action.IndyLifecycleException; import org.commonjava.indy.action.StartupAction; import org.commonjava.indy.data.StoreDataManager; -import org.commonjava.indy.model.core.ArtifactStore; -import org.commonjava.indy.model.core.StoreKey; -import org.commonjava.indy.model.core.StoreType; -import org.commonjava.indy.subsys.infinispan.CacheHandle; import javax.inject.Inject; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; public class InfinispanStoreDataByPkgMapStartupAction implements StartupAction diff --git a/db/infinispan/src/main/java/org/commonjava/indy/infinispan/data/InfinispanStoreDataManager.java b/db/infinispan/src/main/java/org/commonjava/indy/infinispan/data/InfinispanStoreDataManager.java index b3c6bc6cc7..2a03220290 100644 --- a/db/infinispan/src/main/java/org/commonjava/indy/infinispan/data/InfinispanStoreDataManager.java +++ b/db/infinispan/src/main/java/org/commonjava/indy/infinispan/data/InfinispanStoreDataManager.java @@ -345,9 +345,9 @@ else if ( action == STORE ) } public void initAffectedBy() - throws IndyDataException { - streamArtifactStores().filter( s -> group == s.getType() ).forEach( s -> refreshAffectedBy( s, null, STORE ) ); + final Set allStores = getAllArtifactStores(); + allStores.stream().filter( s -> group == s.getType() ).forEach( s -> refreshAffectedBy( s, null, STORE ) ); checkAffectedByCacheHealth(); } diff --git a/db/infinispan/src/main/java/org/commonjava/indy/infinispan/data/InfinispanStoreDataReverseMapStartupAction.java b/db/infinispan/src/main/java/org/commonjava/indy/infinispan/data/InfinispanStoreDataReverseMapStartupAction.java index 0bdd93b865..0fb59c733e 100644 --- a/db/infinispan/src/main/java/org/commonjava/indy/infinispan/data/InfinispanStoreDataReverseMapStartupAction.java +++ b/db/infinispan/src/main/java/org/commonjava/indy/infinispan/data/InfinispanStoreDataReverseMapStartupAction.java @@ -17,19 +17,10 @@ public class InfinispanStoreDataReverseMapStartupAction @Override public void start() - throws IndyLifecycleException { if ( storeDataManager instanceof InfinispanStoreDataManager ) { - try - { - ( (InfinispanStoreDataManager) storeDataManager ).initAffectedBy(); - } - catch ( IndyDataException e ) - { - throw new IndyLifecycleException( "Failed to reverse-map stores and groups they affect: " + e.getMessage(), - e ); - } + ( (InfinispanStoreDataManager) storeDataManager ).initAffectedBy(); } } diff --git a/ftests/core/pom.xml b/ftests/core/pom.xml index a35d9c26cb..4d660deac2 100644 --- a/ftests/core/pom.xml +++ b/ftests/core/pom.xml @@ -28,6 +28,10 @@ Indy :: Functional Tests :: Core + + org.commonjava.indy + indy-db-infinispan + ${project.groupId} indy-client-core-java diff --git a/ftests/core/src/main/java/org/commonjava/indy/ftest/core/content/StoreReverseMapInitTest.java b/ftests/core/src/main/java/org/commonjava/indy/ftest/core/content/StoreReverseMapInitTest.java new file mode 100644 index 0000000000..cc85e44195 --- /dev/null +++ b/ftests/core/src/main/java/org/commonjava/indy/ftest/core/content/StoreReverseMapInitTest.java @@ -0,0 +1,253 @@ +/** + * Copyright (C) 2011-2020 Red Hat, Inc. (https://github.com/Commonjava/indy) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.commonjava.indy.ftest.core.content; + +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang.StringUtils; +import org.commonjava.indy.ftest.core.AbstractContentManagementTest; +import org.commonjava.indy.ftest.core.fixture.StoreTestDataBootupAction; +import org.commonjava.indy.model.core.Group; +import org.commonjava.indy.model.core.HostedRepository; +import org.commonjava.indy.model.core.StoreKey; +import org.custommonkey.xmlunit.XMLUnit; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.xml.sax.SAXException; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; + +import static org.commonjava.indy.ftest.core.fixture.StoreTestDataConstants.groupX; +import static org.commonjava.indy.ftest.core.fixture.StoreTestDataConstants.groupY; +import static org.commonjava.indy.ftest.core.fixture.StoreTestDataConstants.repoA; +import static org.commonjava.indy.ftest.core.fixture.StoreTestDataConstants.repoB; +import static org.commonjava.indy.ftest.core.fixture.StoreTestDataConstants.repoC; +import static org.commonjava.indy.model.core.StoreType.group; +import static org.commonjava.indy.model.core.StoreType.hosted; +import static org.commonjava.indy.pkg.PackageTypeConstants.PKG_TYPE_MAVEN; +import static org.custommonkey.xmlunit.XMLAssert.assertXMLEqual; +import static org.junit.Assert.fail; + +/** + * GIVEN: + *
    + *
  • Hosted repos A,B and C exist
    • + *
  • Hosted repos A, B and C already contain path org/foo/bar/maven-metadata.xml
    • + *
  • Group X contains A, B
    • + *
  • Group Y contains X
    • + *
+ * + * WHEN: + *
    + *
  • Path org/foo/bar/maven-metadata.xml is accessed from Group Y, and has its aggregated metadata generated
    • + *
  • Hosted repo C is appended to the membership of X
    • + *
+ * + * THEN: + *
    + *
  • org/foo/bar/maven-metadata.xml should be removed and regenerated for Y
    • + *
+ * + */ +public class StoreReverseMapInitTest + extends AbstractContentManagementTest +{ + final String path = "org/foo/bar/maven-metadata.xml"; + + @BeforeClass + public static void beforeClass() + { + StoreTestDataBootupAction.enable(); + } + + @AfterClass + public static void afterClass() + { + StoreTestDataBootupAction.disable(); + } + + @Test + public void run() + throws Exception + { + /* @formatter:off */ + final String repoAContent = "\n" + + "\n" + + " org.foo\n" + + " bar\n" + + " \n" + + " 1.0\n" + + " 1.0\n" + + " \n" + + " 1.0\n" + + " \n" + + " 20150721164334\n" + + " \n" + + "\n"; + /* @formatter:on */ + + /* @formatter:off */ + final String repoBContent = "\n" + + "\n" + + " org.foo\n" + + " bar\n" + + " \n" + + " 2.0\n" + + " 2.0\n" + + " \n" + + " 2.0\n" + + " \n" + + " 20160722164334\n" + + " \n" + + "\n"; + /* @formatter:on */ + + /* @formatter:off */ + final String repoCContent = "\n" + + "\n" + + " org.foo\n" + + " bar\n" + + " \n" + + " 3.0\n" + + " 3.0\n" + + " \n" + + " 3.0\n" + + " \n" + + " 20160723164334\n" + + " \n" + + "\n"; + /* @formatter:on */ + + HostedRepository hostedA = client.stores().load( new StoreKey( PKG_TYPE_MAVEN, hosted, repoA ), HostedRepository.class ); + createHostedStorePath( hostedA, repoAContent ); + try (final InputStream stream = client.content().get( hostedA.getKey(), path )) + { + assertContent( repoAContent, IOUtils.toString( stream ) ); + } + + HostedRepository hostedB = client.stores().load( new StoreKey( PKG_TYPE_MAVEN, hosted, repoB ), HostedRepository.class ); + createHostedStorePath( hostedB, repoBContent ); + try (final InputStream stream = client.content().get( hostedB.getKey(), path )) + { + assertContent( repoBContent, IOUtils.toString( stream ) ); + } + + HostedRepository hostedC = client.stores().load( new StoreKey( PKG_TYPE_MAVEN, hosted, repoC ), HostedRepository.class ); + createHostedStorePath( hostedC, repoCContent ); + try (final InputStream stream = client.content().get( hostedC.getKey(), path )) + { + assertContent( repoCContent, IOUtils.toString( stream ) ); + } + + /* @formatter:off */ + final String mergedContent1 = + "\n" + + " \n" + + " org.foo\n" + + " bar\n" + + " \n" + + " 2.0\n" + + " 2.0\n" + + " \n" + + " 1.0\n" + + " 2.0\n" + + " \n" + + " 20160722164334\n" + + " \n" + + "\n"; + /* @formatter:on */ + + Group gY = client.stores().load( new StoreKey( PKG_TYPE_MAVEN, group, groupY ), Group.class ); + try (final InputStream stream = client.content().get( gY.getKey(), path )) + { + assertContent( mergedContent1, IOUtils.toString( stream ) ); + } + + Group gX = client.stores().load( new StoreKey( PKG_TYPE_MAVEN, group, groupX ), Group.class ); + gX.addConstituent( hostedC ); + + client.stores().update( gX, "test update" ); + + gX = client.stores().load( gX.getKey(), Group.class ); + logger.debug( "\n\nGroup constituents are:\n {}\n\n", StringUtils.join( gX.getConstituents(), "\n " ) ); + + waitForEventPropagation(); + + /* @formatter:off */ + final String mergedContent2 = + "\n" + + " \n" + + " org.foo\n" + + " bar\n" + + " \n" + + " 3.0\n" + + " 3.0\n" + + " \n" + + " 1.0\n" + + " 2.0\n" + + " 3.0\n" + + " \n" + + " 20160723164334\n" + + " \n" + + "\n"; + /* @formatter:on */ + + try (final InputStream stream = client.content().get( gY.getKey(), path )) + { + assertContent( mergedContent2, IOUtils.toString( stream ) ); + } + } + + @Override + protected boolean createStandardTestStructures() + { + return false; + } + + private void createHostedStorePath( final HostedRepository hosted, final String content ) + throws Exception + { + if(StringUtils.isNotBlank( content )) + { + client.content().store( hosted.getKey(), path, new ByteArrayInputStream( content.getBytes() ) ); + } + } + + private void assertContent( String expectedXml, String actual ) + throws IOException + { + + logger.debug( "Comparing downloaded XML:\n\n{}\n\nTo expected XML:\n\n{}\n\n", actual, expectedXml ); + + try + { + XMLUnit.setIgnoreWhitespace( true ); + XMLUnit.setIgnoreDiffBetweenTextAndCDATA( true ); + XMLUnit.setIgnoreAttributeOrder( true ); + XMLUnit.setIgnoreComments( true ); + + assertXMLEqual( expectedXml, actual ); + } + catch ( SAXException e ) + { + e.printStackTrace(); + fail( "Downloaded XML not equal to expected XML" ); + } + } +} diff --git a/ftests/core/src/main/java/org/commonjava/indy/ftest/core/fixture/StoreTestDataBootupAction.java b/ftests/core/src/main/java/org/commonjava/indy/ftest/core/fixture/StoreTestDataBootupAction.java new file mode 100644 index 0000000000..4a744f6d1e --- /dev/null +++ b/ftests/core/src/main/java/org/commonjava/indy/ftest/core/fixture/StoreTestDataBootupAction.java @@ -0,0 +1,83 @@ +package org.commonjava.indy.ftest.core.fixture; + +import org.commonjava.indy.action.BootupAction; +import org.commonjava.indy.action.IndyLifecycleException; +import org.commonjava.indy.infinispan.data.StoreDataCache; +import org.commonjava.indy.model.core.ArtifactStore; +import org.commonjava.indy.model.core.Group; +import org.commonjava.indy.model.core.HostedRepository; +import org.commonjava.indy.model.core.StoreKey; +import org.commonjava.indy.subsys.infinispan.CacheHandle; +import org.infinispan.Cache; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.enterprise.context.ApplicationScoped; +import javax.inject.Inject; + +import static org.commonjava.indy.ftest.core.fixture.StoreTestDataConstants.groupX; +import static org.commonjava.indy.ftest.core.fixture.StoreTestDataConstants.groupY; +import static org.commonjava.indy.ftest.core.fixture.StoreTestDataConstants.repoA; +import static org.commonjava.indy.ftest.core.fixture.StoreTestDataConstants.repoB; +import static org.commonjava.indy.ftest.core.fixture.StoreTestDataConstants.repoC; +import static org.commonjava.indy.pkg.PackageTypeConstants.PKG_TYPE_MAVEN; + +@ApplicationScoped +public class StoreTestDataBootupAction + implements BootupAction +{ + @Inject + @StoreDataCache + private CacheHandle stores; + + private static boolean enabled = false; + + public static void enable(){ + enabled = true; + } + + public static void disable(){ + enabled = false; + } + + final String path = "org/foo/bar/maven-metadata.xml"; + + private HostedRepository hostedA = new HostedRepository( PKG_TYPE_MAVEN, repoA ); + private HostedRepository hostedB = new HostedRepository( PKG_TYPE_MAVEN, repoB ); + private HostedRepository hostedC = new HostedRepository( PKG_TYPE_MAVEN, repoC ); + + private Group gX = new Group( PKG_TYPE_MAVEN, groupX, hostedA.getKey(), hostedB.getKey() ); + private Group gY = new Group( PKG_TYPE_MAVEN, groupY, gX.getKey() ); + + @Override + public void init() + throws IndyLifecycleException + { + Logger logger = LoggerFactory.getLogger( getClass() ); + logger.info( "TEST-DATA INIT: initialize store data? {}", enabled ); + + if ( enabled ) + { + stores.put( hostedA.getKey(), hostedA ); + stores.put( hostedB.getKey(), hostedB ); + stores.put( hostedC.getKey(), hostedC ); + stores.put( gX.getKey(), gX ); + stores.put( gY.getKey(), gY ); + + int sz = stores.executeCache( Cache::size ); + logger.info( "TEST-DATA INIT: {} test stores added to the cache.", sz ); + } + } + + @Override + public int getBootPriority() + { + return 10; + } + + @Override + public String getId() + { + return "Store data test initializer"; + } +} diff --git a/ftests/core/src/main/java/org/commonjava/indy/ftest/core/fixture/StoreTestDataConstants.java b/ftests/core/src/main/java/org/commonjava/indy/ftest/core/fixture/StoreTestDataConstants.java new file mode 100644 index 0000000000..bd27ff8167 --- /dev/null +++ b/ftests/core/src/main/java/org/commonjava/indy/ftest/core/fixture/StoreTestDataConstants.java @@ -0,0 +1,13 @@ +package org.commonjava.indy.ftest.core.fixture; + +public final class StoreTestDataConstants +{ + public static final String repoA = "hostedA"; + public static final String repoB = "hostedB"; + public static final String repoC = "hostedC"; + + public static final String groupX = "groupX"; + + public static final String groupY = "groupY"; + +}