NeoStoreFileListing now use StorageEngine to list store files

And also include meta data about the files, such as StoreType and recordSize.

Behaviour change:
Only the count store file currently in use is included in the list.
Previously, all count store files was listed.

community/kernel/src/main/java/org/neo4j/kernel/NeoStoreDataSource.java

@@ -81,6 +81,7 @@
 import org.neo4j.kernel.impl.locking.StatementLocksFactory;
 import org.neo4j.kernel.impl.logging.LogService;
 import org.neo4j.kernel.impl.proc.Procedures;
+import org.neo4j.storageengine.api.StoreFileMetadata;
 import org.neo4j.kernel.impl.storageengine.impl.recordstorage.RecordStorageEngine;
 import org.neo4j.kernel.impl.store.MetaDataStore;
 import org.neo4j.kernel.impl.store.StoreId;
@@ -802,7 +803,7 @@ private KernelModule buildKernel( TransactionAppender appender,
         kernel.registerTransactionHook( transactionEventHandlers );
 
         final NeoStoreFileListing fileListing = new NeoStoreFileListing( storeDir, labelScanStore, indexingService,
-                legacyIndexProviderLookup );
+                legacyIndexProviderLookup, storageEngine );
 
         return new KernelModule()
         {
@@ -949,7 +950,7 @@ public KernelAPI getKernel()
         return kernelModule.kernelAPI();
     }
 
-    public ResourceIterator<File> listStoreFiles( boolean includeLogs ) throws IOException
+    public ResourceIterator<StoreFileMetadata> listStoreFiles( boolean includeLogs ) throws IOException
     {
         return kernelModule.fileListing().listStoreFiles( includeLogs );
     }

community/kernel/src/main/java/org/neo4j/kernel/impl/storageengine/impl/recordstorage/RecordStorageEngine.java

@@ -24,6 +24,7 @@
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
+import java.util.Optional;
 import java.util.function.Supplier;
 
 import org.neo4j.concurrent.WorkSync;
@@ -69,15 +70,20 @@
 import org.neo4j.kernel.impl.index.IndexConfigStore;
 import org.neo4j.kernel.impl.locking.LockGroup;
 import org.neo4j.kernel.impl.locking.LockService;
+import org.neo4j.storageengine.api.StoreFileMetadata;
 import org.neo4j.kernel.impl.storageengine.impl.recordstorage.id.BufferedIdController;
 import org.neo4j.kernel.impl.storageengine.impl.recordstorage.id.DefaultIdController;
 import org.neo4j.kernel.impl.storageengine.impl.recordstorage.id.IdController;
 import org.neo4j.kernel.impl.store.NeoStores;
+import org.neo4j.kernel.impl.store.RecordStore;
 import org.neo4j.kernel.impl.store.SchemaStorage;
 import org.neo4j.kernel.impl.store.StoreFactory;
+import org.neo4j.kernel.impl.store.StoreType;
+import org.neo4j.kernel.impl.store.format.RecordFormat;
 import org.neo4j.kernel.impl.store.id.IdGeneratorFactory;
 import org.neo4j.kernel.impl.store.id.IdReuseEligibility;
 import org.neo4j.kernel.impl.store.id.configuration.IdTypeConfigurationProvider;
+import org.neo4j.kernel.impl.store.record.AbstractBaseRecord;
 import org.neo4j.kernel.impl.transaction.command.CacheInvalidationBatchTransactionApplier;
 import org.neo4j.kernel.impl.transaction.command.HighIdBatchTransactionApplier;
 import org.neo4j.kernel.impl.transaction.command.IndexBatchTransactionApplier;
@@ -497,6 +503,36 @@ public void prepareForRecoveryRequired()
         neoStores.deleteIdGenerators();
     }
 
+    @Override
+    public Collection<StoreFileMetadata> listStorageFiles()
+    {
+        List<StoreFileMetadata> files = new ArrayList<>();
+        for ( StoreType type : StoreType.values() )
+        {
+            if ( type.equals( StoreType.COUNTS ) )
+            {
+                addCurrentCountStoreFile( files );
+            }
+            else
+            {
+                final RecordStore<AbstractBaseRecord> recordStore = neoStores.getRecordStore( type );
+                StoreFileMetadata metadata =
+                        new StoreFileMetadata( recordStore.getStorageFileName(), Optional.of( type ),
+                                recordStore.getRecordSize() );
+                files.add( metadata );
+            }
+        }
+        return files;
+    }
+
+    private void addCurrentCountStoreFile( List<StoreFileMetadata> files )
+    {
+        File countStoreFile = neoStores.getCounts().currentFile();
+        StoreFileMetadata countStoreFileMetadata = new StoreFileMetadata( countStoreFile,
+                Optional.of( StoreType.COUNTS ), RecordFormat.NO_RECORD_SIZE );
+        files.add( countStoreFileMetadata );
+    }
+
     /**
      * @return the underlying {@link NeoStores} which should <strong>ONLY</strong> be accessed by tests
      * until all tests are properly converted to not rely on access to {@link NeoStores}. Currently there

community/kernel/src/main/java/org/neo4j/kernel/impl/store/format/RecordFormat.java

@@ -37,6 +37,8 @@
  */
 public interface RecordFormat<RECORD extends AbstractBaseRecord>
 {
+    int NO_RECORD_SIZE = 1;
+
     /**
      * Instantiates a new record to use in {@link #read(AbstractBaseRecord, PageCursor, RecordLoad, int)}
      * and {@link #write(AbstractBaseRecord, PageCursor, int)}. Records may be reused, which is why the instantiation

community/kernel/src/main/java/org/neo4j/kernel/impl/store/format/standard/NoRecordFormat.java

@@ -40,7 +40,7 @@ public RECORD newRecord()
     @Override
     public int getRecordSize( StoreHeader storeHeader )
     {
-        return 1;
+        return NO_RECORD_SIZE;
     }
 
     @Override

community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/state/NeoStoreFileListing.java

@@ -23,17 +23,21 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
-import java.util.Objects;
+import java.util.Optional;
+import java.util.function.Function;
+import java.util.stream.Collectors;
 
 import org.neo4j.graphdb.Resource;
 import org.neo4j.graphdb.ResourceIterator;
-import org.neo4j.helpers.collection.Iterators;
 import org.neo4j.kernel.api.labelscan.LabelScanStore;
 import org.neo4j.kernel.impl.api.LegacyIndexProviderLookup;
 import org.neo4j.kernel.impl.api.index.IndexingService;
 import org.neo4j.kernel.impl.index.IndexConfigStore;
 import org.neo4j.kernel.impl.store.MetaDataStore;
+import org.neo4j.kernel.impl.store.format.RecordFormat;
 import org.neo4j.kernel.spi.legacyindex.IndexImplementation;
+import org.neo4j.storageengine.api.StorageEngine;
+import org.neo4j.storageengine.api.StoreFileMetadata;
 
 import static java.util.Arrays.asList;
 import static org.neo4j.helpers.collection.Iterators.resourceIterator;
@@ -44,20 +48,25 @@ public class NeoStoreFileListing
     private final LabelScanStore labelScanStore;
     private final IndexingService indexingService;
     private final LegacyIndexProviderLookup legacyIndexProviders;
+    private final StorageEngine storageEngine;
+    private final Function<File,StoreFileMetadata> toNotAStoreTypeFile =
+            file -> new StoreFileMetadata( file, Optional.empty(), RecordFormat.NO_RECORD_SIZE );
 
-    public NeoStoreFileListing( File storeDir, LabelScanStore labelScanStore,
-            IndexingService indexingService, LegacyIndexProviderLookup legacyIndexProviders )
+    public NeoStoreFileListing( File storeDir, LabelScanStore labelScanStore, IndexingService indexingService,
+            LegacyIndexProviderLookup legacyIndexProviders, StorageEngine storageEngine )
     {
         this.storeDir = storeDir;
         this.labelScanStore = labelScanStore;
         this.indexingService = indexingService;
         this.legacyIndexProviders = legacyIndexProviders;
+        this.storageEngine = storageEngine;
     }
 
-    public ResourceIterator<File> listStoreFiles( boolean includeLogs ) throws IOException
+    public ResourceIterator<StoreFileMetadata> listStoreFiles( boolean includeLogs ) throws IOException
     {
-        Collection<File> files = new ArrayList<>();
-        gatherNeoStoreFiles( files, includeLogs );
+        Collection<StoreFileMetadata> files = new ArrayList<>();
+        gatherNeoStoreFiles( files );
+        gatherNonRecordStores( files, includeLogs );
         Resource labelScanStoreSnapshot = gatherLabelScanStoreFiles( files );
         Resource schemaIndexSnapshots = gatherSchemaIndexFiles( files );
         Resource legacyIndexSnapshots = gatherLegacyIndexFiles( files );
@@ -66,77 +75,56 @@ public ResourceIterator<File> listStoreFiles( boolean includeLogs ) throws IOExc
                 new MultiResource( asList( labelScanStoreSnapshot, schemaIndexSnapshots, legacyIndexSnapshots ) ) );
     }
 
-    private Resource gatherLegacyIndexFiles( Collection<File> files ) throws IOException
+    private void gatherNonRecordStores( Collection<StoreFileMetadata> files, boolean includeLogs )
+    {
+        for ( File file : storeDir.listFiles() )
+        {
+            if ( file.getName().equals( IndexConfigStore.INDEX_DB_FILE_NAME ) )
+            {
+                files.add( toNotAStoreTypeFile.apply( file ) );
+            }
+            else if ( includeLogs && transactionLogFile( file.getName() ) )
+            {
+                files.add( toNotAStoreTypeFile.apply( file ) );
+            }
+        }
+    }
+
+    private Resource gatherLegacyIndexFiles( Collection<StoreFileMetadata> files ) throws IOException
     {
         final Collection<ResourceIterator<File>> snapshots = new ArrayList<>();
         for ( IndexImplementation indexProvider : legacyIndexProviders.all() )
         {
             ResourceIterator<File> snapshot = indexProvider.listStoreFiles();
             snapshots.add( snapshot );
-            Iterators.addToCollection( snapshot, files );
+            snapshot.stream().map( toNotAStoreTypeFile ).collect( Collectors.toCollection( () -> files ) );
         }
         // Intentionally don't close the snapshot here, return it for closing by the consumer of
         // the targetFiles list.
         return new MultiResource( snapshots );
     }
 
-    private Resource gatherSchemaIndexFiles(Collection<File> targetFiles) throws IOException
+    private Resource gatherSchemaIndexFiles( Collection<StoreFileMetadata> targetFiles ) throws IOException
     {
         ResourceIterator<File> snapshot = indexingService.snapshotStoreFiles();
-        Iterators.addToCollection(snapshot, targetFiles);
+        snapshot.stream().map( toNotAStoreTypeFile ).collect( Collectors.toCollection( () -> targetFiles ) );
         // Intentionally don't close the snapshot here, return it for closing by the consumer of
         // the targetFiles list.
         return snapshot;
     }
 
-    private Resource gatherLabelScanStoreFiles( Collection<File> targetFiles ) throws IOException
+    private Resource gatherLabelScanStoreFiles( Collection<StoreFileMetadata> targetFiles ) throws IOException
     {
         ResourceIterator<File> snapshot = labelScanStore.snapshotStoreFiles();
-        Iterators.addToCollection(snapshot, targetFiles);
+        snapshot.stream().map( toNotAStoreTypeFile ).collect( Collectors.toCollection( () -> targetFiles ) );
         // Intentionally don't close the snapshot here, return it for closing by the consumer of
         // the targetFiles list.
         return snapshot;
     }
 
-    private void gatherNeoStoreFiles( final Collection<File> targetFiles, boolean includeTransactionLogs )
+    private void gatherNeoStoreFiles( final Collection<StoreFileMetadata> targetFiles )
     {
-        File neostoreFile = null;
-        for ( File dbFile : Objects.requireNonNull( storeDir.listFiles() ) )
-        {
-            String name = dbFile.getName();
-            if ( dbFile.isFile() )
-            {
-                if ( name.equals( MetaDataStore.DEFAULT_NAME ) )
-                {   // Keep it, to add last
-                    neostoreFile = dbFile;
-                }
-                else if ( neoStoreFile( name ) )
-                {
-                    targetFiles.add( dbFile );
-                }
-                else if ( includeTransactionLogs && transactionLogFile( name ) )
-                {
-                    targetFiles.add( dbFile );
-                }
-            }
-        }
-        targetFiles.add( neostoreFile );
-    }
-
-    private boolean neoStoreFile( String name )
-    {
-        if ( name.endsWith( ".id" ) )
-        {
-            return false;
-        }
-
-        if ( name.equals( IndexConfigStore.INDEX_DB_FILE_NAME ) )
-        {
-            return true;
-        }
-
-        return name.startsWith( MetaDataStore.DEFAULT_NAME ) &&
-                !name.startsWith( MetaDataStore.DEFAULT_NAME + ".transaction" );
+        targetFiles.addAll( storageEngine.listStorageFiles() );
     }
 
     private boolean transactionLogFile( String name )

community/kernel/src/main/java/org/neo4j/storageengine/api/StorageEngine.java

@@ -116,6 +116,12 @@ void createCommands(
      */
     void prepareForRecoveryRequired();
 
+    /**
+     * @return a {@link Collection} of {@link StoreFileMetadata} containing metadata about all store files managed by this
+     * {@link StorageEngine}.
+     */
+    Collection<StoreFileMetadata> listStorageFiles();
+
     // ====================================================================
     // All these methods below are temporary while in the process of
     // creating this API, take little notice to them, as they will go away

community/kernel/src/main/java/org/neo4j/storageengine/api/StoreFileMetadata.java

@@ -0,0 +1,54 @@
+/*
+ * Copyright (c) 2002-2016 "Neo Technology,"
+ * Network Engine for Objects in Lund AB [http://neotechnology.com]
+ *
+ * This file is part of Neo4j.
+ *
+ * Neo4j is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.neo4j.storageengine.api;
+
+import java.io.File;
+import java.util.Optional;
+
+import org.neo4j.kernel.impl.store.StoreType;
+
+public class StoreFileMetadata
+{
+    private final File file;
+    private final Optional<StoreType> storeType;
+    private final int recordSize;
+
+    public StoreFileMetadata( File file, Optional<StoreType> storeType, int recordSize )
+    {
+        this.file = file;
+        this.storeType = storeType;
+        this.recordSize = recordSize;
+    }
+
+    public File file()
+    {
+        return file;
+    }
+
+    public Optional<StoreType> storeType()
+    {
+        return storeType;
+    }
+
+    public int recordSize()
+    {
+        return recordSize;
+    }
+}