Commonjava · ruhan1 · Oct 25, 2019 · Oct 25, 2019
diff --git a/...ts/common/src/main/java/org/commonjava/indy/ftest/core/AbstractContentManagementTest.java b/...ts/common/src/main/java/org/commonjava/indy/ftest/core/AbstractContentManagementTest.java
@@ -18,32 +18,39 @@
 import static org.apache.commons.lang3.StringUtils.isNotBlank;
 import static org.commonjava.indy.model.core.StoreType.group;
 import static org.commonjava.indy.model.core.StoreType.remote;
+import static org.commonjava.storage.pathmapped.util.PathMapUtils.getFileId;
+import static org.commonjava.storage.pathmapped.util.PathMapUtils.getStoragePathByFileId;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.CoreMatchers.notNullValue;
 import static org.hamcrest.CoreMatchers.nullValue;
 import static org.junit.Assert.assertThat;
 
 import java.io.BufferedReader;
 import java.io.ByteArrayInputStream;
+import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
+import java.nio.file.Paths;
 import java.util.Arrays;
 
 import org.apache.commons.io.IOUtils;
 import org.apache.maven.artifact.repository.metadata.Metadata;
 import org.apache.maven.artifact.repository.metadata.io.xpp3.MetadataXpp3Reader;
 import org.commonjava.indy.client.core.IndyClientException;
-import org.commonjava.indy.ftest.core.AbstractIndyFunctionalTest;
 import org.commonjava.indy.model.core.ArtifactStore;
 import org.commonjava.indy.model.core.Group;
 import org.commonjava.indy.model.core.HostedRepository;
 import org.commonjava.indy.model.core.RemoteRepository;
 import org.commonjava.indy.model.core.StoreKey;
+import org.commonjava.maven.galley.model.Location;
+import org.commonjava.maven.galley.spi.cache.CacheProvider;
 import org.commonjava.test.http.expect.ExpectationServer;
 import org.junit.Before;
 import org.junit.Rule;
 
+import javax.enterprise.inject.spi.CDI;
+
 public class AbstractContentManagementTest
     extends AbstractIndyFunctionalTest
 {
@@ -116,6 +123,33 @@ protected boolean createStandardTestStructures()
         return true;
     }
 
+    protected File getPhysicalStorageFile( Location location, String path )
+    {
+        String root = fixture.getBootOptions().getHomeDir();
+        String storage = "var/lib/indy/storage";
+        String fileSystem = location.getName();
+        String id = getFileId( fileSystem, path );
+        String hashedPath = getStoragePathByFileId( id );
+
+        File ret = Paths.get( root, storage, hashedPath ).toFile();
+        logger.debug( "Get physical storage file: {}", ret );
+        return ret;
+    }
+
+    protected void sleepAndRunFileGC( long milliseconds )
+    {
+        try
+        {
+            Thread.sleep( milliseconds );
+        }
+        catch ( InterruptedException e )
+        {
+            e.printStackTrace();
+        }
+        CacheProvider cacheProvider = CDI.current().select( CacheProvider.class).get();
+        cacheProvider.asAdminView().gc();
+    }
+
     protected void assertExistence( ArtifactStore store, String path, boolean expected )
             throws IndyClientException
     {

diff --git a/ftests/common/src/main/java/org/commonjava/indy/ftest/core/AbstractIndyFunctionalTest.java b/ftests/common/src/main/java/org/commonjava/indy/ftest/core/AbstractIndyFunctionalTest.java
@@ -68,6 +68,8 @@ public abstract class AbstractIndyFunctionalTest
 
     protected Indy client;
 
+    protected boolean pathMappedStorage = true;
+
     protected CoreServerFixture fixture;
 
     protected final Logger logger = LoggerFactory.getLogger( getClass() );
@@ -111,8 +113,11 @@ public void run()
 
             Thread.currentThread().setName( getClass().getSimpleName() + "." + name.getMethodName() );
 
-            EmbeddedCassandraServerHelper.startEmbeddedCassandra();
-            logger.debug( "Embedded Cassandra server started" );
+            if ( pathMappedStorage )
+            {
+                EmbeddedCassandraServerHelper.startEmbeddedCassandra();
+                logger.debug( "Embedded Cassandra server started" );
+            }
 
             fixture = newServerFixture();
             fixture.start();
@@ -186,7 +191,6 @@ protected int getTestTimeoutMultiplier()
     public void stop()
             throws IndyLifecycleException
     {
-//        waitForEventPropagation();
         closeQuietly( fixture );
         closeQuietly( client );
     }
@@ -228,6 +232,7 @@ protected void initBaseTestConfig( CoreServerFixture fixture )
     {
         writeConfigFile( "conf.d/storage.conf", "[storage-default]\n"
                         + "storage.dir=" + fixture.getBootOptions().getHomeDir() + "/var/lib/indy/storage\n"
+                        + "storage.gc.graceperiodinhours=0\n"
                         + "storage.cassandra.port=9142\n"
                         + "storage.cassandra.keyspace=" + getClass().getSimpleName() );
         if ( isSchedulerEnabled() )

diff --git a/...c/main/java/org/commonjava/indy/ftest/core/content/AbstractContentTimeoutWorkingTest.java b/...c/main/java/org/commonjava/indy/ftest/core/content/AbstractContentTimeoutWorkingTest.java
@@ -18,16 +18,16 @@
 import org.commonjava.indy.client.core.helper.PathInfo;
 import org.commonjava.indy.ftest.core.AbstractContentManagementTest;
 import org.commonjava.indy.model.core.RemoteRepository;
+import org.commonjava.indy.model.galley.KeyedLocation;
+import org.commonjava.indy.util.LocationUtils;
 import org.commonjava.test.http.expect.ExpectationServer;
 import org.junit.Before;
 import org.junit.Rule;
 
 import java.io.File;
-import java.nio.file.Paths;
 import java.util.Date;
 
 import static org.commonjava.indy.model.core.StoreType.remote;
-import static org.commonjava.indy.pkg.maven.model.MavenPackageTypeDescriptor.MAVEN_PKG_KEY;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.CoreMatchers.notNullValue;
 import static org.junit.Assert.assertThat;
@@ -43,6 +43,8 @@ public abstract class AbstractContentTimeoutWorkingTest
     @Rule
     public ExpectationServer server = new ExpectationServer( "repos" );
 
+    protected KeyedLocation location;
+
     @Override
     protected boolean createStandardTestStructures()
     {
@@ -71,6 +73,8 @@ public void setupRepo()
         final String changelog = "Timeout Testing: " + name.getMethodName();
         final RemoteRepository repository = createRemoteRepository( repoId );
 
+        location = LocationUtils.toLocation( repository );
+
         client.stores().create( repository, changelog, RemoteRepository.class );
 
         // ensure the pom exist before the timeout checking
@@ -80,9 +84,12 @@ public void setupRepo()
 
         client.content().get(remote, repoId, pomPath).close(); // force storage
 
+        pomFile = getPhysicalStorageFile( location, pomPath );
+/*
         pomFile =
                 Paths.get( fixture.getBootOptions().getHomeDir(), "var/lib/indy/storage", MAVEN_PKG_KEY, remote.singularEndpointName() + "-" + repoId,
                            pomPath ).toFile();
+*/
 
         assertThat( "pom doesn't exist: " + pomFile, this.pomFile.exists(), equalTo( true ) );
 
@@ -91,8 +98,8 @@ public void setupRepo()
     protected void fileCheckingAfterTimeout()
             throws Exception
     {
-        // make sure the repo timout
-        Thread.sleep( getTestTimeoutMultiplier() * TIMEOUT_WAITING_MILLISECONDS );
+        // make sure the repo timeout
+        sleepAndRunFileGC( getTestTimeoutMultiplier() * TIMEOUT_WAITING_MILLISECONDS );
         logger.debug( "Timeout time {}s passed!", TIMEOUT_SECONDS );
 
         assertThat( "artifact should be removed when timeout", pomFile.exists(), equalTo( false ) );

diff --git a/.../main/java/org/commonjava/indy/ftest/core/content/AbstractMetadataTimeoutWorkingTest.java b/.../main/java/org/commonjava/indy/ftest/core/content/AbstractMetadataTimeoutWorkingTest.java
@@ -18,6 +18,8 @@
 import org.commonjava.indy.client.core.helper.PathInfo;
 import org.commonjava.indy.ftest.core.AbstractContentManagementTest;
 import org.commonjava.indy.model.core.RemoteRepository;
+import org.commonjava.indy.model.galley.KeyedLocation;
+import org.commonjava.indy.util.LocationUtils;
 import org.commonjava.test.http.expect.ExpectationServer;
 import org.junit.Before;
 import org.junit.Rule;
@@ -44,6 +46,8 @@ public abstract class AbstractMetadataTimeoutWorkingTest
     @Rule
     public ExpectationServer server = new ExpectationServer( "repos" );
 
+    protected KeyedLocation location;
+
     protected File pomFile;
 
     protected File metadataFile;
@@ -83,6 +87,8 @@ public void setupRepo()
         final String changelog = "Timeout Testing: " + name.getMethodName();
         final RemoteRepository repository = createRemoteRepository();
 
+        location = LocationUtils.toLocation( repository );
+
         client.stores().create( repository, changelog, RemoteRepository.class );
 
         // ensure the pom exist before the timeout checking
@@ -91,8 +97,11 @@ public void setupRepo()
         assertThat( "no pom result", pomResult, notNullValue() );
         assertThat( "pom doesn't exist", pomResult.exists(), equalTo( true ) );
 
+        pomFile = getPhysicalStorageFile( location, pomPath );
+/*
         pomFile = Paths.get( fixture.getBootOptions().getHomeDir(), "var/lib/indy/storage", MAVEN_PKG_KEY,
                              remote.singularEndpointName() + "-" + repoId, pomPath ).toFile();
+*/
 
         assertThat( "pom doesn't exist: " + pomFile, pomFile.exists(), equalTo( true ) );
 
@@ -102,8 +111,11 @@ public void setupRepo()
         assertThat( "no metadata result", metadataResult, notNullValue() );
         assertThat( "metadata doesn't exist", metadataResult.exists(), equalTo( true ) );
 
+        metadataFile = getPhysicalStorageFile( location, metadataPath );
+/*
         metadataFile = Paths.get( fixture.getBootOptions().getHomeDir(), "var/lib/indy/storage", MAVEN_PKG_KEY,
                                   remote.singularEndpointName() + "-" + repoId, metadataPath ).toFile();
+*/
 
         assertThat( "metadata doesn't exist", metadataFile.exists(), equalTo( true ) );
 
@@ -113,8 +125,11 @@ public void setupRepo()
         assertThat( "no archetype result", archetypeResult, notNullValue() );
         assertThat( "archetype doesn't exist", archetypeResult.exists(), equalTo( true ) );
 
+        archetypeFile = getPhysicalStorageFile( location, archetypePath );
+/*
         archetypeFile = Paths.get( fixture.getBootOptions().getHomeDir(), "var/lib/indy/storage", MAVEN_PKG_KEY,
                                   remote.singularEndpointName() + "-" + repoId, archetypePath ).toFile();
+*/
 
         assertThat( "archetype doesn't exist: " + archetypeFile, archetypeFile.exists(), equalTo( true ) );
 

diff --git a/...re/src/main/java/org/commonjava/indy/ftest/core/content/CacheFirstTimeoutWorkingTest.java b/...re/src/main/java/org/commonjava/indy/ftest/core/content/CacheFirstTimeoutWorkingTest.java
@@ -15,13 +15,11 @@
  */
 package org.commonjava.indy.ftest.core.content;
 
-import org.commonjava.indy.ftest.core.category.EventDependent;
 import org.commonjava.indy.ftest.core.category.TimingDependent;
 import org.commonjava.indy.model.core.RemoteRepository;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 
-import java.io.File;
 import java.util.Date;
 
 import static org.hamcrest.CoreMatchers.equalTo;
@@ -50,7 +48,7 @@ public void timeout()
         logger.debug( "Starting sleep at: {}", new Date() );
 
         // make sure the non-metadata content times out
-        Thread.sleep( CACHE_TIMEOUT_WAITING_MILLISECONDS );
+        sleepAndRunFileGC( CACHE_TIMEOUT_WAITING_MILLISECONDS );
         logger.debug( "Verifying content timeouts at: {} (timeout: {}s)", new Date(), CACHE_TIMEOUT_SECONDS );
 
         assertThat( "artifact should be removed when cache timeout", pomFile.exists(), equalTo( false ) );
@@ -59,7 +57,7 @@ public void timeout()
                     equalTo( true ) );
 
         // make sure the metadata content times out
-        Thread.sleep( METADATA_TIMEOUT_ADDITIONAL_WAITING_MILLISECONDS );
+        sleepAndRunFileGC( METADATA_TIMEOUT_ADDITIONAL_WAITING_MILLISECONDS );
         logger.debug( "Verifying metadata timeouts at: {} (timeout: {}s)", new Date(), METADATA_TIMEOUT_SECONDS );
 
         assertThat( "metadata should be removed when metadata timeout", metadataFile.exists(), equalTo( false ) );

diff --git a/...src/main/java/org/commonjava/indy/ftest/core/content/MetadataFirstTimeoutWorkingTest.java b/...src/main/java/org/commonjava/indy/ftest/core/content/MetadataFirstTimeoutWorkingTest.java
@@ -15,14 +15,11 @@
  */
 package org.commonjava.indy.ftest.core.content;
 
-import org.commonjava.indy.ftest.core.category.EventDependent;
 import org.commonjava.indy.ftest.core.category.TimingDependent;
 import org.commonjava.indy.model.core.RemoteRepository;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 
-import java.io.File;
-
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.junit.Assert.assertThat;
 /**
@@ -46,15 +43,15 @@ public void timeout()
             throws Exception
     {
         // make sure the metadata timout
-        Thread.sleep( METADATA_TIMEOUT_WAITING_MILLISECONDS );
+        sleepAndRunFileGC( METADATA_TIMEOUT_WAITING_MILLISECONDS );
         logger.debug( "Timeout time {}s passed!", METADATA_TIMEOUT_SECONDS );
         assertThat( "metadata should be removed when metadata timeout", metadataFile.exists(), equalTo( false ) );
         assertThat( "archetype should be removed when metadata timeout", archetypeFile.exists(),
                     equalTo( false ) );
         assertThat( "artifact should not be removed when metadata timeout", pomFile.exists(), equalTo( true ) );
 
         // make sure the repo timout
-        Thread.sleep( CACHE_TIMEOUT_WAITING_MILLISECONDS );
+        sleepAndRunFileGC( CACHE_TIMEOUT_WAITING_MILLISECONDS );
         logger.debug( "Timeout time {}s passed!", CACHE_TIMEOUT_SECONDS );
         assertThat( "artifact should be removed when cache timeout", pomFile.exists(), equalTo( false ) );
     }

diff --git a/...in/java/org/commonjava/indy/ftest/core/content/MetadataPassthroughTimeoutWorkingTest.java b/...in/java/org/commonjava/indy/ftest/core/content/MetadataPassthroughTimeoutWorkingTest.java
@@ -15,7 +15,6 @@
  */
 package org.commonjava.indy.ftest.core.content;
 
-import org.commonjava.indy.ftest.core.category.EventDependent;
 import org.commonjava.indy.ftest.core.category.TimingDependent;
 import org.commonjava.indy.model.core.RemoteRepository;
 import org.commonjava.indy.test.fixture.core.CoreServerFixture;
@@ -61,19 +60,19 @@ protected void initTestConfig( CoreServerFixture fixture )
     public void timeout()
             throws Exception
     {
-        Thread.sleep( CACHE_TIMEOUT_WAITING_MILLISECONDS );
+        sleepAndRunFileGC( CACHE_TIMEOUT_WAITING_MILLISECONDS );
         logger.debug( "Verifying content timeouts at: {} (timeout: {}s)", new Date(), CACHE_TIMEOUT_SECONDS );
         assertThat( "artifact should not be removed because of passthrough", pomFile.exists(), equalTo( true ) );
         assertThat( "metadata should not be removed because of passthrough", metadataFile.exists(), equalTo( true ) );
         assertThat( "archetype should not be removed because of passthrough", archetypeFile.exists(), equalTo( true ) );
 
-        Thread.sleep( METADATA_TIMEOUT_WAITING_MILLISECONDS );
+        sleepAndRunFileGC( METADATA_TIMEOUT_WAITING_MILLISECONDS );
         logger.debug( "Verifying metadata timeouts at: {} (timeout: {}s)", new Date(), METADATA_TIMEOUT_SECONDS );
         assertThat( "artifact should not be removed because of passthrough", pomFile.exists(), equalTo( true ) );
         assertThat( "metadata should not be removed because of passthrough", metadataFile.exists(), equalTo( true ) );
         assertThat( "archetype should not be removed because of passthrough", archetypeFile.exists(), equalTo( true ) );
 
-        Thread.sleep( PASSTHROUGH_TIMEOUT_WAITING_MILLISECONDS );
+        sleepAndRunFileGC( PASSTHROUGH_TIMEOUT_WAITING_MILLISECONDS );
         logger.debug( "Verifying passthrough timeouts at: {} (timeout: {}s)", new Date(), PASSTHROUGH_TIMEOUT_SECONDS );
         assertThat( "artifact should be removed because of passthrough timeout", pomFile.exists(), equalTo( false ) );
         assertThat( "metadata should be removed because of passthrough timeout", metadataFile.exists(),

diff --git a/ftests/core/src/main/java/org/commonjava/indy/ftest/core/content/MetadataTimeoutTest.java b/ftests/core/src/main/java/org/commonjava/indy/ftest/core/content/MetadataTimeoutTest.java
@@ -21,7 +21,6 @@
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 
-import static java.lang.Thread.sleep;
 import static org.commonjava.indy.model.core.StoreType.remote;
 import static org.commonjava.indy.pkg.maven.model.MavenPackageTypeDescriptor.MAVEN_PKG_KEY;
 import static org.hamcrest.CoreMatchers.equalTo;
@@ -42,13 +41,13 @@ public class MetadataTimeoutTest
     public void timeout() throws Exception
     {
         // make the metadata.xml timeout
-        sleep( METADATA_TIMEOUT_WAITING_MILLISECONDS );
+        sleepAndRunFileGC( METADATA_TIMEOUT_WAITING_MILLISECONDS );
         assertThat( "Metadata not removed when timeout", metadataFile.exists(), equalTo( false ) );
 
         // retrieve it again
         client.content().get( new StoreKey( MAVEN_PKG_KEY, remote, repoId ), metadataPath ).close();
 
-        sleep( METADATA_TIMEOUT_WAITING_MILLISECONDS );
+        sleepAndRunFileGC( METADATA_TIMEOUT_WAITING_MILLISECONDS );
         assertThat( "(Second) Metadata not removed when timeout", metadataFile.exists(), equalTo( false ) );
     }