Cleanups around stress tests

community/io/src/test/java/org/neo4j/io/pagecache/impl/muninn/MuninnPageCacheStressIT.java

@@ -21,10 +21,13 @@
 
 import org.junit.Test;
 
-import org.neo4j.io.pagecache.tracing.DefaultPageCacheTracer;
+import java.io.File;
+
 import org.neo4j.io.pagecache.stress.Condition;
 import org.neo4j.io.pagecache.stress.PageCacheStressTest;
+import org.neo4j.io.pagecache.tracing.DefaultPageCacheTracer;
 
+import static java.lang.System.getProperty;
 import static org.neo4j.io.pagecache.stress.Conditions.numberOfEvictions;
 
 /**
@@ -43,6 +46,7 @@ public void shouldHandleTheStressOfOneMillionEvictions() throws Exception
         Condition condition = numberOfEvictions( monitor, 1_000_000 );
 
         PageCacheStressTest runner = new PageCacheStressTest.Builder()
+                .withWorkingDirectory( new File( getProperty( "java.io.tmpdir" ) ) )
                 .with( monitor )
                 .with( condition )
                 .build();

community/io/src/test/java/org/neo4j/io/pagecache/stress/PageCacheStressTest.java

@@ -19,6 +19,8 @@
  */
 package org.neo4j.io.pagecache.stress;
 
+import java.io.File;
+
 import org.neo4j.io.fs.DefaultFileSystemAbstraction;
 import org.neo4j.io.pagecache.PageCache;
 import org.neo4j.io.pagecache.PageSwapperFactory;
@@ -60,7 +62,7 @@ public class PageCacheStressTest
     private final PageCacheTracer tracer;
     private final Condition condition;
 
-    private final String workingDirectory;
+    private final File workingDirectory;
 
     private PageCacheStressTest( Builder builder )
     {
@@ -102,7 +104,7 @@ public static class Builder
         PageCacheTracer tracer = NULL;
         Condition condition;
 
-        String workingDirectory = getProperty( "java.io.tmpdir" );
+        File workingDirectory;
 
         public PageCacheStressTest build()
         {
@@ -141,7 +143,7 @@ public Builder withNumberOfCachePages( int numberOfCachePages )
             return this;
         }
 
-        public Builder withWorkingDirectory( String workingDirectory )
+        public Builder withWorkingDirectory( File workingDirectory )
         {
             this.workingDirectory = workingDirectory;
             return this;

community/io/src/test/java/org/neo4j/io/pagecache/stress/PageCacheStresser.java

@@ -22,7 +22,6 @@
 import java.io.File;
 import java.io.IOException;
 import java.nio.file.Files;
-import java.nio.file.Paths;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.concurrent.ExecutionException;
@@ -52,9 +51,9 @@ public class PageCacheStresser
     private final int maxPages;
     private final int numberOfThreads;
 
-    private final String workingDirectory;
+    private final File workingDirectory;
 
-    public PageCacheStresser( int maxPages, int numberOfThreads, String workingDirectory )
+    public PageCacheStresser( int maxPages, int numberOfThreads, File workingDirectory )
     {
         this.maxPages = maxPages;
         this.numberOfThreads = numberOfThreads;
@@ -63,7 +62,7 @@ public PageCacheStresser( int maxPages, int numberOfThreads, String workingDirec
 
     public void stress( PageCache pageCache, Condition condition ) throws Exception
     {
-        File file = Files.createTempFile( Paths.get( workingDirectory ), "pagecacheundertest", ".bin" ).toFile();
+        File file = Files.createTempFile( workingDirectory.toPath(), "pagecacheundertest", ".bin" ).toFile();
         file.deleteOnExit();
 
         int cachePageSize = pageCache.pageSize();

stresstests/src/test/java/org/neo4j/StressTestingHelper.java

@@ -0,0 +1,53 @@
+/*
+ * Copyright (c) 2002-2016 "Neo Technology,"
+ * Network Engine for Objects in Lund AB [http://neotechnology.com]
+ *
+ * This file is part of Neo4j.
+ *
+ * Neo4j is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.neo4j;
+
+import java.io.File;
+
+import static java.lang.System.getenv;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+public class StressTestingHelper
+{
+    private StressTestingHelper()
+    {
+    }
+
+    public static File ensureExistsAndEmpty( File directory )
+    {
+        if ( !directory.mkdirs() )
+        {
+            assertTrue( directory.exists() );
+            assertTrue( directory.isDirectory() );
+            String[] list = directory.list();
+            assertNotNull( list );
+            assertEquals( 0, list.length );
+        }
+        return directory;
+    }
+
+    public static String fromEnv( String environmentVariableName, String defaultValue )
+    {
+        String environmentVariableValue = getenv( environmentVariableName );
+        return environmentVariableValue == null ? defaultValue : environmentVariableValue;
+    }
+}

stresstests/src/test/java/org/neo4j/backup/stresstests/BackupServiceStressTesting.java

@@ -34,6 +34,8 @@
 import static java.lang.System.getenv;
 import static java.util.concurrent.TimeUnit.MINUTES;
 import static org.junit.Assert.assertEquals;
+import static org.neo4j.StressTestingHelper.ensureExistsAndEmpty;
+import static org.neo4j.StressTestingHelper.fromEnv;
 import static org.neo4j.backup.BackupServiceStressTestingBuilder.untilTimeExpired;
 
 /**
@@ -58,32 +60,17 @@ public void shouldBehaveCorrectlyUnderStress() throws Exception
         File workDirectory = new File( directory, "work" );
         Callable<Integer> callable = new BackupServiceStressTestingBuilder()
                 .until( untilTimeExpired( durationInMinutes, MINUTES ) )
-                .withStore( ensureExists( storeDirectory ) )
-                .withBackupDirectory( ensureExists( workDirectory ) )
+                .withStore( ensureExistsAndEmpty( storeDirectory ) )
+                .withBackupDirectory( ensureExistsAndEmpty( workDirectory ) )
                 .withBackupAddress( backupHostname, backupPort )
                 .build();
 
         int brokenStores = callable.call();
 
         assertEquals( 0, brokenStores );
 
+        // let's cleanup disk space when everything went well
         FileUtils.deleteRecursively( storeDirectory );
         FileUtils.deleteRecursively( workDirectory );
     }
-
-    private static File ensureExists( File directory ) throws IOException
-    {
-        FileUtils.deleteRecursively( directory );
-        if ( !directory.mkdirs() )
-        {
-            throw new IOException( "Unable to create directory: '" + directory.getAbsolutePath() + "'" );
-        }
-        return directory;
-    }
-
-    private static String fromEnv( String environmentVariableName, String defaultValue )
-    {
-        String environmentVariableValue = getenv( environmentVariableName );
-        return environmentVariableValue == null ? defaultValue : environmentVariableValue;
-    }
 }

stresstests/src/test/java/org/neo4j/index/population/LucenePartitionedIndexStressTesting.java

@@ -19,7 +19,6 @@
  */
 package org.neo4j.index.population;
 
-import org.apache.commons.lang3.StringUtils;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
@@ -53,6 +52,7 @@
 import static org.apache.commons.lang3.SystemUtils.JAVA_IO_TMPDIR;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
+import static org.neo4j.StressTestingHelper.fromEnv;
 
 public class LucenePartitionedIndexStressTesting
 {
@@ -63,16 +63,16 @@ public class LucenePartitionedIndexStressTesting
     private static final int NUMBER_OF_PROPERTIES = 2;
 
     private static final int NUMBER_OF_POPULATORS =
-            Integer.valueOf( getEnvVariable( "LUCENE_INDEX_NUMBER_OF_POPULATORS",
+            Integer.valueOf( fromEnv( "LUCENE_INDEX_NUMBER_OF_POPULATORS",
                     String.valueOf( Runtime.getRuntime().availableProcessors() - 1 ) ) );
-    private static final int BATCH_SIZE = Integer.valueOf( getEnvVariable( "LUCENE_INDEX_POPULATION_BATCH_SIZE",
+    private static final int BATCH_SIZE = Integer.valueOf( fromEnv( "LUCENE_INDEX_POPULATION_BATCH_SIZE",
             String.valueOf( 10000 ) ) );
 
-    private static final long NUMBER_OF_NODES = Long.valueOf( getEnvVariable(
+    private static final long NUMBER_OF_NODES = Long.valueOf( fromEnv(
             "LUCENE_PARTITIONED_INDEX_NUMBER_OF_NODES", String.valueOf( 100000 ) ) );
     private static final String WORK_DIRECTORY =
-            getEnvVariable( "LUCENE_PARTITIONED_INDEX_WORKING_DIRECTORY", JAVA_IO_TMPDIR );
-    private static final int WAIT_DURATION_MINUTES = Integer.valueOf( getEnvVariable(
+            fromEnv( "LUCENE_PARTITIONED_INDEX_WORKING_DIRECTORY", JAVA_IO_TMPDIR );
+    private static final int WAIT_DURATION_MINUTES = Integer.valueOf( fromEnv(
             "LUCENE_PARTITIONED_INDEX_WAIT_TILL_ONLINE", String.valueOf( 30 ) ) );
 
     private ExecutorService populators;
@@ -265,12 +265,6 @@ private static String getUniqueStringProperty()
         return UNIQUE_PROPERTY_PREFIX + 0;
     }
 
-    private static String getEnvVariable( String propertyName, String defaultValue )
-    {
-        String value = System.getenv( propertyName );
-        return StringUtils.defaultIfEmpty( value, defaultValue );
-    }
-
     private static class SequentialStringSupplier implements Supplier<String>
     {
         private final int step;

stresstests/src/test/java/org/neo4j/io/pagecache/stresstests/PageCacheStressTesting.java

@@ -21,13 +21,17 @@
 
 import org.junit.Test;
 
-import org.neo4j.io.pagecache.tracing.DefaultPageCacheTracer;
+import java.io.File;
+
+import org.neo4j.io.fs.FileUtils;
 import org.neo4j.io.pagecache.stress.PageCacheStressTest;
+import org.neo4j.io.pagecache.tracing.DefaultPageCacheTracer;
 
 import static java.lang.Integer.parseInt;
 import static java.lang.System.getProperty;
-import static java.lang.System.getenv;
 import static java.util.concurrent.TimeUnit.MINUTES;
+import static org.neo4j.StressTestingHelper.ensureExistsAndEmpty;
+import static org.neo4j.StressTestingHelper.fromEnv;
 import static org.neo4j.io.pagecache.stress.Conditions.timePeriod;
 
 /**
@@ -45,21 +49,21 @@ public class PageCacheStressTesting
     @Test
     public void shouldBehaveCorrectlyUnderStress() throws Exception
     {
-        int durationInMinutes = parseInt( fromEnvironmentOrDefault( "PAGE_CACHE_STRESS_DURATION", "1" ) );
-        int numberOfPages = parseInt( fromEnvironmentOrDefault( "PAGE_CACHE_STRESS_NUMBER_OF_PAGES", "10000" ) );
-        int numberOfThreads = parseInt( fromEnvironmentOrDefault( "PAGE_CACHE_STRESS_NUMBER_OF_THREADS", "8" ) );
-        int numberOfCachePages = parseInt( fromEnvironmentOrDefault( "PAGE_CACHE_STRESS_NUMBER_OF_CACHE_PAGES", "1000" ) );
+        int durationInMinutes = parseInt( fromEnv( "PAGE_CACHE_STRESS_DURATION", "1" ) );
+        int numberOfPages = parseInt( fromEnv( "PAGE_CACHE_STRESS_NUMBER_OF_PAGES", "10000" ) );
+        int numberOfThreads = parseInt( fromEnv( "PAGE_CACHE_STRESS_NUMBER_OF_THREADS", "8" ) );
+        int numberOfCachePages = parseInt( fromEnv( "PAGE_CACHE_STRESS_NUMBER_OF_CACHE_PAGES", "1000" ) );
+        File baseDir = new File( fromEnv( "PAGE_CACHE_STRESS_WORKING_DIRECTORY", getProperty( "java.io.tmpdir" ) ) );
 
-        String workingDirectory = fromEnvironmentOrDefault( "PAGE_CACHE_STRESS_WORKING_DIRECTORY", getProperty( "java.io.tmpdir" ) );
+        File workingDirectory = new File( baseDir,  "working" );
 
         DefaultPageCacheTracer monitor = new DefaultPageCacheTracer();
-
         PageCacheStressTest runner = new PageCacheStressTest.Builder()
                 .with( timePeriod( durationInMinutes, MINUTES ) )
                 .withNumberOfPages( numberOfPages )
                 .withNumberOfThreads( numberOfThreads )
                 .withNumberOfCachePages( numberOfCachePages )
-                .withWorkingDirectory( workingDirectory )
+                .withWorkingDirectory( ensureExistsAndEmpty( workingDirectory ) )
                 .with( monitor )
                 .build();
 
@@ -72,17 +76,8 @@ public void shouldBehaveCorrectlyUnderStress() throws Exception
         long flushes = monitor.flushes();
         System.out.printf( " - page faults: %d%n - evictions: %d%n - pins: %d%n - unpins: %d%n - flushes: %d%n",
                 faults, evictions, pins, unpins, flushes );
-    }
-
-    private static String fromEnvironmentOrDefault( String environmentVariableName, String defaultValue )
-    {
-        String environmentVariableValue = getenv( environmentVariableName );
-
-        if ( environmentVariableValue == null )
-        {
-            return defaultValue;
-        }
 
-        return environmentVariableValue;
+        // let's cleanup disk space when everything went well
+        FileUtils.deleteRecursively( workingDirectory );
     }
 }

stresstests/src/test/java/org/neo4j/kernel/stresstests/transaction/checkpoint/CheckPointingLogRotationStressTesting.java

@@ -40,7 +40,8 @@
 import static java.lang.Integer.parseInt;
 import static java.lang.Long.parseLong;
 import static java.lang.System.getProperty;
-import static java.lang.System.getenv;
+import static org.neo4j.StressTestingHelper.ensureExistsAndEmpty;
+import static org.neo4j.StressTestingHelper.fromEnv;
 import static org.neo4j.kernel.stresstests.transaction.checkpoint.mutation.RandomMutationFactory.defaultRandomMutation;
 import static org.neo4j.unsafe.impl.batchimport.Configuration.DEFAULT;
 
@@ -63,20 +64,15 @@ public void shouldBehaveCorrectlyUnderStress() throws Throwable
     {
         long durationInMinutes =
                 parseLong( fromEnv( "CHECK_POINT_LOG_ROTATION_STRESS_DURATION", DEFAULT_DURATION_IN_MINUTES ) );
-        File storeDir = ensureExists( fromEnv( "CHECK_POINT_LOG_ROTATION_STORE_DIRECTORY", DEFAULT_STORE_DIR ) );
+        File storeDir = new File( fromEnv( "CHECK_POINT_LOG_ROTATION_STORE_DIRECTORY", DEFAULT_STORE_DIR ) );
         long nodeCount = parseLong( fromEnv( "CHECK_POINT_LOG_ROTATION_NODE_COUNT", DEFAULT_NODE_COUNT ) );
         int threads = parseInt( fromEnv( "CHECK_POINT_LOG_ROTATION_WORKER_THREADS", DEFAULT_WORKER_THREADS ) );
         String pageCacheMemory = fromEnv( "CHECK_POINT_LOG_ROTATION_PAGE_CACHE_MEMORY", DEFAULT_PAGE_CACHE_MEMORY );
         String pageSize = fromEnv( "CHECK_POINT_LOG_ROTATION_PAGE_SIZE", DEFAULT_PAGE_SIZE );
 
-        if ( storeDir.exists() )
-        {
-            FileUtils.deleteRecursively( storeDir );
-        }
-
         System.out.println( "1/6\tBuilding initial store..." );
-        new ParallelBatchImporter( storeDir, DEFAULT, NullLogService.getInstance(), ExecutionMonitors.defaultVisible(),
-                Config.defaults() ).doImport( new NodeCountInputs( nodeCount ) );
+        new ParallelBatchImporter( ensureExistsAndEmpty( storeDir ), DEFAULT, NullLogService.getInstance(),
+                ExecutionMonitors.defaultVisible(), Config.defaults() ).doImport( new NodeCountInputs( nodeCount ) );
 
         System.out.println( "2/6\tStarting database..." );
         GraphDatabaseService db = new GraphDatabaseFactory().newEmbeddedDatabaseBuilder( storeDir )
@@ -115,19 +111,8 @@ public void shouldBehaveCorrectlyUnderStress() throws Throwable
         {
             System.out.println( "Done." );
         }
-    }
 
-    private File ensureExists( String directory )
-    {
-        File dir = new File( directory );
-        //noinspection ResultOfMethodCallIgnored
-        dir.mkdirs();
-        return dir;
-    }
-
-    private static String fromEnv( String environmentVariableName, String defaultValue )
-    {
-        String environmentVariableValue = getenv( environmentVariableName );
-        return environmentVariableValue == null ? defaultValue : environmentVariableValue;
+        // let's cleanup disk space when everything went well
+        FileUtils.deleteRecursively( storeDir );
     }
 }