Do not force store file snapshot on udc approximate store size calcul…

…ation Before this PR each UDC report of approximate store size and that caused snapshoting of all the indexes in the store with by itself can cause quite a load in case if you have lots of them, also provoking locking issues with lucene. This PR will change that behaviour to simply count database directory size instead.
neo4j · Oct 4, 2018 · bed98bc · bed98bc
1 parent f5855f8
commit bed98bc
Show file tree

Hide file tree

Showing 6 changed files with 69 additions and 105 deletions.
diff --git a/community/io/src/main/java/org/neo4j/io/fs/DefaultFileSystemAbstraction.java b/community/io/src/main/java/org/neo4j/io/fs/DefaultFileSystemAbstraction.java
@@ -127,9 +127,9 @@ public boolean fileExists( File file )
     }
 
     @Override
-    public long getFileSize( File fileName )
+    public long getFileSize( File file )
     {
-        return fileName.length();
+        return file.length();
     }
 
     @Override

diff --git a/community/io/src/main/java/org/neo4j/io/fs/FileUtils.java b/community/io/src/main/java/org/neo4j/io/fs/FileUtils.java
@@ -713,16 +713,16 @@ public static OutputStream openAsOutputStream( Path path, boolean append ) throw
      * Calculates the size of a given directory or file given the provided abstract filesystem.
      *
      * @param fs the filesystem abstraction to use
-     * @param path to the file or directory.
+     * @param file to the file or directory.
      * @return the size, in bytes, of the file or the total size of the content in the directory, including
      * subdirectories.
      */
-    public static long size( FileSystemAbstraction fs, File path )
+    public static long size( FileSystemAbstraction fs, File file )
     {
-        if ( fs.isDirectory( path ) )
+        if ( fs.isDirectory( file ) )
         {
             long size = 0L;
-            File[] files = fs.listFiles( path );
+            File[] files = fs.listFiles( file );
             if ( files == null )
             {
                 return 0L;
@@ -735,7 +735,7 @@ public static long size( FileSystemAbstraction fs, File path )
         }
         else
         {
-            return fs.getFileSize( path );
+            return fs.getFileSize( file );
         }
     }
 }
diff --git a/...ity/kernel/src/main/java/org/neo4j/kernel/impl/transaction/state/NeoStoreFileListing.java b/...ity/kernel/src/main/java/org/neo4j/kernel/impl/transaction/state/NeoStoreFileListing.java
@@ -26,7 +26,6 @@
 import java.util.List;
 import java.util.concurrent.CopyOnWriteArraySet;
 import java.util.function.Function;
-import java.util.stream.Collectors;
 
 import org.neo4j.graphdb.Resource;
 import org.neo4j.graphdb.ResourceIterator;
@@ -303,11 +302,6 @@ public ResourceIterator<StoreFileMetadata> build() throws IOException
         }
     }
 
-    public static List<StoreFileMetadata> getSnapshotFilesMetadata( ResourceIterator<File> snapshot )
-    {
-        return snapshot.stream().map( toNotAStoreTypeFile ).collect( Collectors.toList() );
-    }
-
     private void gatherNeoStoreFiles( final Collection<StoreFileMetadata> targetFiles )
     {
         targetFiles.addAll( storageEngine.listStorageFiles() );

diff --git a/community/udc/pom.xml b/community/udc/pom.xml
@@ -69,10 +69,6 @@
             <artifactId>commons-io</artifactId>
             <scope>test</scope>
         </dependency>
-        <dependency>
-            <groupId>org.junit.jupiter</groupId>
-            <artifactId>junit-jupiter-migrationsupport</artifactId>
-        </dependency>
         <dependency>
             <groupId>org.junit.vintage</groupId>
             <artifactId>junit-vintage-engine</artifactId>
@@ -81,6 +77,13 @@
             <groupId>org.junit.jupiter</groupId>
             <artifactId>junit-jupiter-engine</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.neo4j</groupId>
+            <artifactId>neo4j-common</artifactId>
+            <version>${project.version}</version>
+            <type>test-jar</type>
+            <scope>test</scope>
+        </dependency>
         <dependency>
             <groupId>org.neo4j</groupId>
             <artifactId>neo4j-kernel</artifactId>

diff --git a/community/udc/src/main/java/org/neo4j/ext/udc/impl/DefaultUdcInformationCollector.java b/community/udc/src/main/java/org/neo4j/ext/udc/impl/DefaultUdcInformationCollector.java
@@ -20,7 +20,6 @@
 package org.neo4j.ext.udc.impl;
 
 import java.io.File;
-import java.io.IOException;
 import java.lang.management.ManagementFactory;
 import java.lang.management.RuntimeMXBean;
 import java.net.InetAddress;
@@ -33,16 +32,17 @@
 import java.util.regex.Pattern;
 
 import org.neo4j.ext.udc.UdcSettings;
-import org.neo4j.graphdb.ResourceIterator;
+import org.neo4j.graphdb.DependencyResolver;
 import org.neo4j.graphdb.config.Setting;
 import org.neo4j.helpers.collection.MapUtil;
+import org.neo4j.io.fs.FileSystemAbstraction;
+import org.neo4j.io.fs.FileUtils;
 import org.neo4j.io.os.OsBeanUtil;
 import org.neo4j.kernel.NeoStoreDataSource;
 import org.neo4j.kernel.configuration.Config;
 import org.neo4j.kernel.impl.store.id.IdGeneratorFactory;
 import org.neo4j.kernel.impl.store.id.IdType;
 import org.neo4j.kernel.impl.transaction.state.DataSourceManager;
-import org.neo4j.storageengine.api.StoreFileMetadata;
 import org.neo4j.udc.UsageData;
 import org.neo4j.udc.UsageDataKeys;
 
@@ -173,32 +173,13 @@ private void addStoreFileSizes( Map<String,String> udcFields )
         {
             return;
         }
-
+        DependencyResolver dependencyResolver = neoStoreDataSource.getDependencyResolver();
-        try ( ResourceIterator<StoreFileMetadata> files = neoStoreDataSource.listStoreFiles( false ) )
+        FileSystemAbstraction fileSystem = dependencyResolver.resolveDependency( FileSystemAbstraction.class );
-        {
+        long databaseSize = FileUtils.size( fileSystem, neoStoreDataSource.getDatabaseLayout().databaseDirectory() );
-            addStoreFileSizes( udcFields, files );
+        add( udcFields, STORE_SIZE, databaseSize );
-        }
-        catch ( IOException | NullPointerException ignored )
-        {
-            // we just didn't report store file sizes this time around
-        }
-    }
-
-    private void addStoreFileSizes( Map<String,String> udcFields, ResourceIterator<StoreFileMetadata> files )
-    {
-        long size = 0;
-
-        while ( files.hasNext() )
-        {
-            File file = files.next().file();
-
-            size += file.length();
-        }
-
-        add( udcFields, STORE_SIZE, size );
     }
 
-    private String determineOsDistribution()
+    private static String determineOsDistribution()
     {
         if ( System.getProperties().getProperty( "os.name", "" ).equals( "Linux" ) )
         {
@@ -245,7 +226,7 @@ private Integer determineClusterNameHash()
         }
     }
 
-    private String determineTags( Map<String,String> jarNamesForTags, String classPath )
+    private static String determineTags( Map<String,String> jarNamesForTags, String classPath )
     {
         StringBuilder result = new StringBuilder();
         for ( Map.Entry<String,String> entry : jarNamesForTags.entrySet() )
@@ -263,13 +244,13 @@ private String determineTags( Map<String,String> jarNamesForTags, String classPa
         return result.substring( 1 );
     }
 
-    private String getClassPath()
+    private static String getClassPath()
     {
         RuntimeMXBean runtime = ManagementFactory.getRuntimeMXBean();
         return runtime.getClassPath();
     }
 
-    private String determineMacAddress()
+    private static String determineMacAddress()
     {
         String formattedMac = "0";
         try
@@ -299,17 +280,17 @@ private String determineMacAddress()
         return formattedMac;
     }
 
-    private int determineNumberOfProcessors()
+    private static int determineNumberOfProcessors()
     {
         return Runtime.getRuntime().availableProcessors();
     }
 
-    private long determineTotalMemory()
+    private static long determineTotalMemory()
     {
         return OsBeanUtil.getTotalPhysicalMemory();
     }
 
-    private long determineHeapSize()
+    private static long determineHeapSize()
     {
         return ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getUsed();
     }
@@ -339,7 +320,7 @@ private long getNumberOfIdsInUse( IdType type )
         return neoStoreDataSource.getDependencyResolver().resolveDependency( IdGeneratorFactory.class ).get( type ).getNumberOfIdsInUse();
     }
 
-    private String toCommaString( Object values )
+    private static String toCommaString( Object values )
     {
         StringBuilder result = new StringBuilder();
         if ( values instanceof Iterable )
@@ -364,7 +345,7 @@ private String toCommaString( Object values )
         return result.toString();
     }
 
-    private void add( Map<String,String> udcFields, String name, Object value )
+    private static void add( Map<String,String> udcFields, String name, Object value )
     {
         if ( value == null )
         {
@@ -378,7 +359,7 @@ private void add( Map<String,String> udcFields, String name, Object value )
         udcFields.put( name, str );
     }
 
-    private String removeUdcPrefix( String propertyName )
+    private static String removeUdcPrefix( String propertyName )
     {
         if ( propertyName.startsWith( UDC_PROPERTY_PREFIX ) )
         {
@@ -387,12 +368,12 @@ private String removeUdcPrefix( String propertyName )
         return propertyName;
     }
 
-    private String sanitizeUdcProperty( String propertyValue )
+    private static String sanitizeUdcProperty( String propertyValue )
     {
         return propertyValue.replace( ' ', '_' );
     }
 
-    private Map<String,String> determineSystemProperties()
+    private static Map<String,String> determineSystemProperties()
     {
         Map<String,String> relevantSysProps = new HashMap<>();
         Properties sysProps = System.getProperties();