diff --git a/ftests/common/src/main/java/org/commonjava/indy/ftest/core/AbstractContentManagementTest.java b/ftests/common/src/main/java/org/commonjava/indy/ftest/core/AbstractContentManagementTest.java index 80188b8e66..ed36105d79 100644 --- a/ftests/common/src/main/java/org/commonjava/indy/ftest/core/AbstractContentManagementTest.java +++ b/ftests/common/src/main/java/org/commonjava/indy/ftest/core/AbstractContentManagementTest.java @@ -18,6 +18,8 @@ import static org.apache.commons.lang3.StringUtils.isNotBlank; import static org.commonjava.indy.model.core.StoreType.group; import static org.commonjava.indy.model.core.StoreType.remote; +import static org.commonjava.storage.pathmapped.util.PathMapUtils.getFileId; +import static org.commonjava.storage.pathmapped.util.PathMapUtils.getStoragePathByFileId; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.notNullValue; import static org.hamcrest.CoreMatchers.nullValue; @@ -25,25 +27,30 @@ import java.io.BufferedReader; import java.io.ByteArrayInputStream; +import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; +import java.nio.file.Paths; import java.util.Arrays; import org.apache.commons.io.IOUtils; import org.apache.maven.artifact.repository.metadata.Metadata; import org.apache.maven.artifact.repository.metadata.io.xpp3.MetadataXpp3Reader; import org.commonjava.indy.client.core.IndyClientException; -import org.commonjava.indy.ftest.core.AbstractIndyFunctionalTest; import org.commonjava.indy.model.core.ArtifactStore; import org.commonjava.indy.model.core.Group; import org.commonjava.indy.model.core.HostedRepository; import org.commonjava.indy.model.core.RemoteRepository; import org.commonjava.indy.model.core.StoreKey; +import org.commonjava.maven.galley.model.Location; +import org.commonjava.maven.galley.spi.cache.CacheProvider; import org.commonjava.test.http.expect.ExpectationServer; import org.junit.Before; import org.junit.Rule; +import javax.enterprise.inject.spi.CDI; + public class AbstractContentManagementTest extends AbstractIndyFunctionalTest { @@ -116,6 +123,33 @@ protected boolean createStandardTestStructures() return true; } + protected File getPhysicalStorageFile( Location location, String path ) + { + String root = fixture.getBootOptions().getHomeDir(); + String storage = "var/lib/indy/storage"; + String fileSystem = location.getName(); + String id = getFileId( fileSystem, path ); + String hashedPath = getStoragePathByFileId( id ); + + File ret = Paths.get( root, storage, hashedPath ).toFile(); + logger.debug( "Get physical storage file: {}", ret ); + return ret; + } + + protected void sleepAndRunFileGC( long milliseconds ) + { + try + { + Thread.sleep( milliseconds ); + } + catch ( InterruptedException e ) + { + e.printStackTrace(); + } + CacheProvider cacheProvider = CDI.current().select( CacheProvider.class).get(); + cacheProvider.asAdminView().gc(); + } + protected void assertExistence( ArtifactStore store, String path, boolean expected ) throws IndyClientException { diff --git a/ftests/common/src/main/java/org/commonjava/indy/ftest/core/AbstractIndyFunctionalTest.java b/ftests/common/src/main/java/org/commonjava/indy/ftest/core/AbstractIndyFunctionalTest.java index bd6dc8146d..4ebfe89b2b 100644 --- a/ftests/common/src/main/java/org/commonjava/indy/ftest/core/AbstractIndyFunctionalTest.java +++ b/ftests/common/src/main/java/org/commonjava/indy/ftest/core/AbstractIndyFunctionalTest.java @@ -68,6 +68,8 @@ public abstract class AbstractIndyFunctionalTest protected Indy client; + protected boolean pathMappedStorage = true; + protected CoreServerFixture fixture; protected final Logger logger = LoggerFactory.getLogger( getClass() ); @@ -111,8 +113,11 @@ public void run() Thread.currentThread().setName( getClass().getSimpleName() + "." + name.getMethodName() ); - EmbeddedCassandraServerHelper.startEmbeddedCassandra(); - logger.debug( "Embedded Cassandra server started" ); + if ( pathMappedStorage ) + { + EmbeddedCassandraServerHelper.startEmbeddedCassandra(); + logger.debug( "Embedded Cassandra server started" ); + } fixture = newServerFixture(); fixture.start(); @@ -186,7 +191,6 @@ protected int getTestTimeoutMultiplier() public void stop() throws IndyLifecycleException { -// waitForEventPropagation(); closeQuietly( fixture ); closeQuietly( client ); } @@ -228,6 +232,7 @@ protected void initBaseTestConfig( CoreServerFixture fixture ) { writeConfigFile( "conf.d/storage.conf", "[storage-default]\n" + "storage.dir=" + fixture.getBootOptions().getHomeDir() + "/var/lib/indy/storage\n" + + "storage.gc.graceperiodinhours=0\n" + "storage.cassandra.port=9142\n" + "storage.cassandra.keyspace=" + getClass().getSimpleName() ); if ( isSchedulerEnabled() ) diff --git a/ftests/core/src/main/java/org/commonjava/indy/ftest/core/content/AbstractContentTimeoutWorkingTest.java b/ftests/core/src/main/java/org/commonjava/indy/ftest/core/content/AbstractContentTimeoutWorkingTest.java index f3ceba0583..62e4e4ee99 100644 --- a/ftests/core/src/main/java/org/commonjava/indy/ftest/core/content/AbstractContentTimeoutWorkingTest.java +++ b/ftests/core/src/main/java/org/commonjava/indy/ftest/core/content/AbstractContentTimeoutWorkingTest.java @@ -18,16 +18,16 @@ import org.commonjava.indy.client.core.helper.PathInfo; import org.commonjava.indy.ftest.core.AbstractContentManagementTest; import org.commonjava.indy.model.core.RemoteRepository; +import org.commonjava.indy.model.galley.KeyedLocation; +import org.commonjava.indy.util.LocationUtils; import org.commonjava.test.http.expect.ExpectationServer; import org.junit.Before; import org.junit.Rule; import java.io.File; -import java.nio.file.Paths; import java.util.Date; import static org.commonjava.indy.model.core.StoreType.remote; -import static org.commonjava.indy.pkg.maven.model.MavenPackageTypeDescriptor.MAVEN_PKG_KEY; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.notNullValue; import static org.junit.Assert.assertThat; @@ -43,6 +43,8 @@ public abstract class AbstractContentTimeoutWorkingTest @Rule public ExpectationServer server = new ExpectationServer( "repos" ); + protected KeyedLocation location; + @Override protected boolean createStandardTestStructures() { @@ -71,6 +73,8 @@ public void setupRepo() final String changelog = "Timeout Testing: " + name.getMethodName(); final RemoteRepository repository = createRemoteRepository( repoId ); + location = LocationUtils.toLocation( repository ); + client.stores().create( repository, changelog, RemoteRepository.class ); // ensure the pom exist before the timeout checking @@ -80,9 +84,12 @@ public void setupRepo() client.content().get(remote, repoId, pomPath).close(); // force storage + pomFile = getPhysicalStorageFile( location, pomPath ); +/* pomFile = Paths.get( fixture.getBootOptions().getHomeDir(), "var/lib/indy/storage", MAVEN_PKG_KEY, remote.singularEndpointName() + "-" + repoId, pomPath ).toFile(); +*/ assertThat( "pom doesn't exist: " + pomFile, this.pomFile.exists(), equalTo( true ) ); @@ -91,8 +98,8 @@ public void setupRepo() protected void fileCheckingAfterTimeout() throws Exception { - // make sure the repo timout - Thread.sleep( getTestTimeoutMultiplier() * TIMEOUT_WAITING_MILLISECONDS ); + // make sure the repo timeout + sleepAndRunFileGC( getTestTimeoutMultiplier() * TIMEOUT_WAITING_MILLISECONDS ); logger.debug( "Timeout time {}s passed!", TIMEOUT_SECONDS ); assertThat( "artifact should be removed when timeout", pomFile.exists(), equalTo( false ) ); diff --git a/ftests/core/src/main/java/org/commonjava/indy/ftest/core/content/AbstractMetadataTimeoutWorkingTest.java b/ftests/core/src/main/java/org/commonjava/indy/ftest/core/content/AbstractMetadataTimeoutWorkingTest.java index 9b992e2c22..03d606bcb9 100644 --- a/ftests/core/src/main/java/org/commonjava/indy/ftest/core/content/AbstractMetadataTimeoutWorkingTest.java +++ b/ftests/core/src/main/java/org/commonjava/indy/ftest/core/content/AbstractMetadataTimeoutWorkingTest.java @@ -18,6 +18,8 @@ import org.commonjava.indy.client.core.helper.PathInfo; import org.commonjava.indy.ftest.core.AbstractContentManagementTest; import org.commonjava.indy.model.core.RemoteRepository; +import org.commonjava.indy.model.galley.KeyedLocation; +import org.commonjava.indy.util.LocationUtils; import org.commonjava.test.http.expect.ExpectationServer; import org.junit.Before; import org.junit.Rule; @@ -44,6 +46,8 @@ public abstract class AbstractMetadataTimeoutWorkingTest @Rule public ExpectationServer server = new ExpectationServer( "repos" ); + protected KeyedLocation location; + protected File pomFile; protected File metadataFile; @@ -83,6 +87,8 @@ public void setupRepo() final String changelog = "Timeout Testing: " + name.getMethodName(); final RemoteRepository repository = createRemoteRepository(); + location = LocationUtils.toLocation( repository ); + client.stores().create( repository, changelog, RemoteRepository.class ); // ensure the pom exist before the timeout checking @@ -91,8 +97,11 @@ public void setupRepo() assertThat( "no pom result", pomResult, notNullValue() ); assertThat( "pom doesn't exist", pomResult.exists(), equalTo( true ) ); + pomFile = getPhysicalStorageFile( location, pomPath ); +/* pomFile = Paths.get( fixture.getBootOptions().getHomeDir(), "var/lib/indy/storage", MAVEN_PKG_KEY, remote.singularEndpointName() + "-" + repoId, pomPath ).toFile(); +*/ assertThat( "pom doesn't exist: " + pomFile, pomFile.exists(), equalTo( true ) ); @@ -102,8 +111,11 @@ public void setupRepo() assertThat( "no metadata result", metadataResult, notNullValue() ); assertThat( "metadata doesn't exist", metadataResult.exists(), equalTo( true ) ); + metadataFile = getPhysicalStorageFile( location, metadataPath ); +/* metadataFile = Paths.get( fixture.getBootOptions().getHomeDir(), "var/lib/indy/storage", MAVEN_PKG_KEY, remote.singularEndpointName() + "-" + repoId, metadataPath ).toFile(); +*/ assertThat( "metadata doesn't exist", metadataFile.exists(), equalTo( true ) ); @@ -113,8 +125,11 @@ public void setupRepo() assertThat( "no archetype result", archetypeResult, notNullValue() ); assertThat( "archetype doesn't exist", archetypeResult.exists(), equalTo( true ) ); + archetypeFile = getPhysicalStorageFile( location, archetypePath ); +/* archetypeFile = Paths.get( fixture.getBootOptions().getHomeDir(), "var/lib/indy/storage", MAVEN_PKG_KEY, remote.singularEndpointName() + "-" + repoId, archetypePath ).toFile(); +*/ assertThat( "archetype doesn't exist: " + archetypeFile, archetypeFile.exists(), equalTo( true ) ); diff --git a/ftests/core/src/main/java/org/commonjava/indy/ftest/core/content/CacheFirstTimeoutWorkingTest.java b/ftests/core/src/main/java/org/commonjava/indy/ftest/core/content/CacheFirstTimeoutWorkingTest.java index 7488135d36..bad34997aa 100644 --- a/ftests/core/src/main/java/org/commonjava/indy/ftest/core/content/CacheFirstTimeoutWorkingTest.java +++ b/ftests/core/src/main/java/org/commonjava/indy/ftest/core/content/CacheFirstTimeoutWorkingTest.java @@ -15,13 +15,11 @@ */ package org.commonjava.indy.ftest.core.content; -import org.commonjava.indy.ftest.core.category.EventDependent; import org.commonjava.indy.ftest.core.category.TimingDependent; import org.commonjava.indy.model.core.RemoteRepository; import org.junit.Test; import org.junit.experimental.categories.Category; -import java.io.File; import java.util.Date; import static org.hamcrest.CoreMatchers.equalTo; @@ -50,7 +48,7 @@ public void timeout() logger.debug( "Starting sleep at: {}", new Date() ); // make sure the non-metadata content times out - Thread.sleep( CACHE_TIMEOUT_WAITING_MILLISECONDS ); + sleepAndRunFileGC( CACHE_TIMEOUT_WAITING_MILLISECONDS ); logger.debug( "Verifying content timeouts at: {} (timeout: {}s)", new Date(), CACHE_TIMEOUT_SECONDS ); assertThat( "artifact should be removed when cache timeout", pomFile.exists(), equalTo( false ) ); @@ -59,7 +57,7 @@ public void timeout() equalTo( true ) ); // make sure the metadata content times out - Thread.sleep( METADATA_TIMEOUT_ADDITIONAL_WAITING_MILLISECONDS ); + sleepAndRunFileGC( METADATA_TIMEOUT_ADDITIONAL_WAITING_MILLISECONDS ); logger.debug( "Verifying metadata timeouts at: {} (timeout: {}s)", new Date(), METADATA_TIMEOUT_SECONDS ); assertThat( "metadata should be removed when metadata timeout", metadataFile.exists(), equalTo( false ) ); diff --git a/ftests/core/src/main/java/org/commonjava/indy/ftest/core/content/MetadataFirstTimeoutWorkingTest.java b/ftests/core/src/main/java/org/commonjava/indy/ftest/core/content/MetadataFirstTimeoutWorkingTest.java index 9b5490171d..c5d8bf7874 100644 --- a/ftests/core/src/main/java/org/commonjava/indy/ftest/core/content/MetadataFirstTimeoutWorkingTest.java +++ b/ftests/core/src/main/java/org/commonjava/indy/ftest/core/content/MetadataFirstTimeoutWorkingTest.java @@ -15,14 +15,11 @@ */ package org.commonjava.indy.ftest.core.content; -import org.commonjava.indy.ftest.core.category.EventDependent; import org.commonjava.indy.ftest.core.category.TimingDependent; import org.commonjava.indy.model.core.RemoteRepository; import org.junit.Test; import org.junit.experimental.categories.Category; -import java.io.File; - import static org.hamcrest.CoreMatchers.equalTo; import static org.junit.Assert.assertThat; /** @@ -46,7 +43,7 @@ public void timeout() throws Exception { // make sure the metadata timout - Thread.sleep( METADATA_TIMEOUT_WAITING_MILLISECONDS ); + sleepAndRunFileGC( METADATA_TIMEOUT_WAITING_MILLISECONDS ); logger.debug( "Timeout time {}s passed!", METADATA_TIMEOUT_SECONDS ); assertThat( "metadata should be removed when metadata timeout", metadataFile.exists(), equalTo( false ) ); assertThat( "archetype should be removed when metadata timeout", archetypeFile.exists(), @@ -54,7 +51,7 @@ public void timeout() assertThat( "artifact should not be removed when metadata timeout", pomFile.exists(), equalTo( true ) ); // make sure the repo timout - Thread.sleep( CACHE_TIMEOUT_WAITING_MILLISECONDS ); + sleepAndRunFileGC( CACHE_TIMEOUT_WAITING_MILLISECONDS ); logger.debug( "Timeout time {}s passed!", CACHE_TIMEOUT_SECONDS ); assertThat( "artifact should be removed when cache timeout", pomFile.exists(), equalTo( false ) ); } diff --git a/ftests/core/src/main/java/org/commonjava/indy/ftest/core/content/MetadataPassthroughTimeoutWorkingTest.java b/ftests/core/src/main/java/org/commonjava/indy/ftest/core/content/MetadataPassthroughTimeoutWorkingTest.java index 4d4fe1c984..0c46e5e5a0 100644 --- a/ftests/core/src/main/java/org/commonjava/indy/ftest/core/content/MetadataPassthroughTimeoutWorkingTest.java +++ b/ftests/core/src/main/java/org/commonjava/indy/ftest/core/content/MetadataPassthroughTimeoutWorkingTest.java @@ -15,7 +15,6 @@ */ package org.commonjava.indy.ftest.core.content; -import org.commonjava.indy.ftest.core.category.EventDependent; import org.commonjava.indy.ftest.core.category.TimingDependent; import org.commonjava.indy.model.core.RemoteRepository; import org.commonjava.indy.test.fixture.core.CoreServerFixture; @@ -61,19 +60,19 @@ protected void initTestConfig( CoreServerFixture fixture ) public void timeout() throws Exception { - Thread.sleep( CACHE_TIMEOUT_WAITING_MILLISECONDS ); + sleepAndRunFileGC( CACHE_TIMEOUT_WAITING_MILLISECONDS ); logger.debug( "Verifying content timeouts at: {} (timeout: {}s)", new Date(), CACHE_TIMEOUT_SECONDS ); assertThat( "artifact should not be removed because of passthrough", pomFile.exists(), equalTo( true ) ); assertThat( "metadata should not be removed because of passthrough", metadataFile.exists(), equalTo( true ) ); assertThat( "archetype should not be removed because of passthrough", archetypeFile.exists(), equalTo( true ) ); - Thread.sleep( METADATA_TIMEOUT_WAITING_MILLISECONDS ); + sleepAndRunFileGC( METADATA_TIMEOUT_WAITING_MILLISECONDS ); logger.debug( "Verifying metadata timeouts at: {} (timeout: {}s)", new Date(), METADATA_TIMEOUT_SECONDS ); assertThat( "artifact should not be removed because of passthrough", pomFile.exists(), equalTo( true ) ); assertThat( "metadata should not be removed because of passthrough", metadataFile.exists(), equalTo( true ) ); assertThat( "archetype should not be removed because of passthrough", archetypeFile.exists(), equalTo( true ) ); - Thread.sleep( PASSTHROUGH_TIMEOUT_WAITING_MILLISECONDS ); + sleepAndRunFileGC( PASSTHROUGH_TIMEOUT_WAITING_MILLISECONDS ); logger.debug( "Verifying passthrough timeouts at: {} (timeout: {}s)", new Date(), PASSTHROUGH_TIMEOUT_SECONDS ); assertThat( "artifact should be removed because of passthrough timeout", pomFile.exists(), equalTo( false ) ); assertThat( "metadata should be removed because of passthrough timeout", metadataFile.exists(), diff --git a/ftests/core/src/main/java/org/commonjava/indy/ftest/core/content/MetadataTimeoutTest.java b/ftests/core/src/main/java/org/commonjava/indy/ftest/core/content/MetadataTimeoutTest.java index e14e0f9377..53249446cb 100644 --- a/ftests/core/src/main/java/org/commonjava/indy/ftest/core/content/MetadataTimeoutTest.java +++ b/ftests/core/src/main/java/org/commonjava/indy/ftest/core/content/MetadataTimeoutTest.java @@ -21,7 +21,6 @@ import org.junit.Test; import org.junit.experimental.categories.Category; -import static java.lang.Thread.sleep; import static org.commonjava.indy.model.core.StoreType.remote; import static org.commonjava.indy.pkg.maven.model.MavenPackageTypeDescriptor.MAVEN_PKG_KEY; import static org.hamcrest.CoreMatchers.equalTo; @@ -42,13 +41,13 @@ public class MetadataTimeoutTest public void timeout() throws Exception { // make the metadata.xml timeout - sleep( METADATA_TIMEOUT_WAITING_MILLISECONDS ); + sleepAndRunFileGC( METADATA_TIMEOUT_WAITING_MILLISECONDS ); assertThat( "Metadata not removed when timeout", metadataFile.exists(), equalTo( false ) ); // retrieve it again client.content().get( new StoreKey( MAVEN_PKG_KEY, remote, repoId ), metadataPath ).close(); - sleep( METADATA_TIMEOUT_WAITING_MILLISECONDS ); + sleepAndRunFileGC( METADATA_TIMEOUT_WAITING_MILLISECONDS ); assertThat( "(Second) Metadata not removed when timeout", metadataFile.exists(), equalTo( false ) ); }