diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/storageengine/impl/recordstorage/RecordStorageEngine.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/storageengine/impl/recordstorage/RecordStorageEngine.java index 43cf295c82517..c09ac86d14429 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/storageengine/impl/recordstorage/RecordStorageEngine.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/storageengine/impl/recordstorage/RecordStorageEngine.java @@ -24,7 +24,6 @@ import java.util.Collection; import java.util.Collections; import java.util.List; -import java.util.Optional; import java.util.function.Supplier; import org.neo4j.concurrent.WorkSync; @@ -524,8 +523,7 @@ public Collection listStorageFiles() { final RecordStore recordStore = neoStores.getRecordStore( type ); StoreFileMetadata metadata = - new StoreFileMetadata( recordStore.getStorageFileName(), Optional.of( type ), - recordStore.getRecordSize() ); + new StoreFileMetadata( recordStore.getStorageFileName(), recordStore.getRecordSize() ); files.add( metadata ); } } @@ -538,7 +536,7 @@ private void addCountStoreFiles( List files ) for ( File countStoreFile : countStoreFiles ) { StoreFileMetadata countStoreFileMetadata = new StoreFileMetadata( countStoreFile, - Optional.of( StoreType.COUNTS ), RecordFormat.NO_RECORD_SIZE ); + RecordFormat.NO_RECORD_SIZE ); files.add( countStoreFileMetadata ); } } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/StoreType.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/StoreType.java index 03c939dde592f..a35ff380d9a7a 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/StoreType.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/StoreType.java @@ -174,6 +174,13 @@ public String getStoreName() { return StoreFactory.COUNTS_STORE; } + + @Override + protected boolean isStoreFile( String fileName) + { + return matchStoreName( fileName, getStoreName() + CountsTracker.RIGHT ) || + matchStoreName( fileName, getStoreName() + CountsTracker.LEFT ); + } }, META_DATA( StoreFile.NEO_STORE ) // Make sure this META store is last { @@ -232,7 +239,7 @@ public static Optional typeOf( String fileName ) StoreType[] values = StoreType.values(); for ( StoreType value : values ) { - if ( fileName.equals( MetaDataStore.DEFAULT_NAME + value.getStoreName() ) ) + if ( value.isStoreFile( fileName ) ) { return Optional.of( value ); } @@ -252,4 +259,14 @@ public static boolean shouldBeManagedByPageCache( String storeFileName ) boolean isLabelScanStore = NativeLabelScanStore.FILE_NAME.equals( storeFileName ); return isLabelScanStore || StoreType.typeOf( storeFileName ).map( StoreType::isRecordStore ).orElse( false ); } + + protected boolean isStoreFile( String fileName ) + { + return matchStoreName( fileName, getStoreName() ); + } + + protected boolean matchStoreName( String fileName, String storeName ) + { + return fileName.equals( MetaDataStore.DEFAULT_NAME + storeName ); + } } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/LogFiles.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/LogFiles.java index 4f57c5d7a9caa..7d38be33f46ff 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/LogFiles.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/LogFiles.java @@ -33,7 +33,7 @@ public class LogFiles { public static final FilenameFilter FILENAME_FILTER = new LogicalLogFilenameFilter(); - public static final class LogicalLogFilenameFilter implements FilenameFilter + private static final class LogicalLogFilenameFilter implements FilenameFilter { private static final Pattern LOG_FILENAME_PATTERN = compile( PhysicalLogFile.REGEX_DEFAULT_NAME + PhysicalLogFile.REGEX_DEFAULT_VERSION_SUFFIX + ".*" diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/state/NeoStoreFileListing.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/state/NeoStoreFileListing.java index 463b9f785a9eb..ec7080e232eda 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/state/NeoStoreFileListing.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/state/NeoStoreFileListing.java @@ -23,6 +23,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collection; +import java.util.List; import java.util.Optional; import java.util.function.Function; import java.util.stream.Collectors; @@ -34,6 +35,7 @@ import org.neo4j.kernel.impl.api.index.IndexingService; import org.neo4j.kernel.impl.index.IndexConfigStore; import org.neo4j.kernel.impl.store.MetaDataStore; +import org.neo4j.kernel.impl.store.StoreType; import org.neo4j.kernel.impl.store.format.RecordFormat; import org.neo4j.kernel.spi.legacyindex.IndexImplementation; import org.neo4j.storageengine.api.StorageEngine; @@ -50,7 +52,7 @@ public class NeoStoreFileListing private final LegacyIndexProviderLookup legacyIndexProviders; private final StorageEngine storageEngine; private final Function toNotAStoreTypeFile = - file -> new StoreFileMetadata( file, Optional.empty(), RecordFormat.NO_RECORD_SIZE ); + file -> new StoreFileMetadata( file, RecordFormat.NO_RECORD_SIZE ); public NeoStoreFileListing( File storeDir, LabelScanStore labelScanStore, IndexingService indexingService, LegacyIndexProviderLookup legacyIndexProviders, StorageEngine storageEngine ) @@ -64,17 +66,38 @@ public NeoStoreFileListing( File storeDir, LabelScanStore labelScanStore, Indexi public ResourceIterator listStoreFiles( boolean includeLogs ) throws IOException { - Collection files = new ArrayList<>(); + List files = new ArrayList<>(); gatherNonRecordStores( files, includeLogs ); gatherNeoStoreFiles( files ); Resource labelScanStoreSnapshot = gatherLabelScanStoreFiles( files ); Resource schemaIndexSnapshots = gatherSchemaIndexFiles( files ); Resource legacyIndexSnapshots = gatherLegacyIndexFiles( files ); + placeMetaDataStoreLast( files ); + return resourceIterator( files.iterator(), new MultiResource( asList( labelScanStoreSnapshot, schemaIndexSnapshots, legacyIndexSnapshots ) ) ); } + private void placeMetaDataStoreLast( List files ) + { + int index = 0; + for ( StoreFileMetadata file : files ) + { + Optional storeType = StoreType.typeOf( file.file().getName() ); + if ( storeType.isPresent() && storeType.get().equals( StoreType.META_DATA ) ) + { + break; + } + index++; + } + if ( index < files.size() - 1 ) + { + StoreFileMetadata metaDataStoreFile = files.remove( index ); + files.add( metaDataStoreFile ); + } + } + private void gatherNonRecordStores( Collection files, boolean includeLogs ) { final File[] storeFiles = storeDir.listFiles(); diff --git a/community/kernel/src/main/java/org/neo4j/storageengine/api/StoreFileMetadata.java b/community/kernel/src/main/java/org/neo4j/storageengine/api/StoreFileMetadata.java index b554386c73ed8..78392c5f84890 100644 --- a/community/kernel/src/main/java/org/neo4j/storageengine/api/StoreFileMetadata.java +++ b/community/kernel/src/main/java/org/neo4j/storageengine/api/StoreFileMetadata.java @@ -20,20 +20,15 @@ package org.neo4j.storageengine.api; import java.io.File; -import java.util.Optional; - -import org.neo4j.kernel.impl.store.StoreType; public class StoreFileMetadata { private final File file; - private final Optional storeType; private final int recordSize; - public StoreFileMetadata( File file, Optional storeType, int recordSize ) + public StoreFileMetadata( File file, int recordSize ) { this.file = file; - this.storeType = storeType; this.recordSize = recordSize; } @@ -42,11 +37,6 @@ public File file() return file; } - public Optional storeType() - { - return storeType; - } - public int recordSize() { return recordSize; diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/storageengine/impl/recordstorage/RecordStorageEngineTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/storageengine/impl/recordstorage/RecordStorageEngineTest.java index 2ce63a975a4be..182387e167715 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/storageengine/impl/recordstorage/RecordStorageEngineTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/storageengine/impl/recordstorage/RecordStorageEngineTest.java @@ -25,10 +25,14 @@ import java.io.File; import java.io.IOException; +import java.util.Arrays; import java.util.Collection; import java.util.Optional; +import java.util.Set; import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Function; +import java.util.stream.Collectors; import org.neo4j.helpers.Exceptions; import org.neo4j.io.fs.FileSystemAbstraction; @@ -39,12 +43,9 @@ import org.neo4j.kernel.impl.api.BatchTransactionApplierFacade; import org.neo4j.kernel.impl.api.CountsAccessor; import org.neo4j.kernel.impl.api.TransactionToApply; -import org.neo4j.kernel.impl.store.StoreFile; import org.neo4j.kernel.impl.store.StoreType; import org.neo4j.kernel.impl.store.UnderlyingStorageException; import org.neo4j.kernel.impl.store.counts.CountsTracker; -import org.neo4j.kernel.impl.store.format.RecordFormat; -import org.neo4j.kernel.impl.storemigration.StoreFileType; import org.neo4j.kernel.impl.transaction.TransactionRepresentation; import org.neo4j.kernel.impl.transaction.log.FakeCommitment; import org.neo4j.kernel.impl.transaction.log.TransactionIdStore; @@ -56,13 +57,11 @@ import org.neo4j.test.rule.RecordStorageEngineRule; import org.neo4j.test.rule.fs.EphemeralFileSystemRule; -import static org.hamcrest.Matchers.anyOf; -import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.sameInstance; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertSame; import static org.junit.Assert.assertThat; -import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import static org.mockito.Matchers.any; import static org.mockito.Mockito.doThrow; @@ -83,6 +82,12 @@ public class RecordStorageEngineTest .around( pageCacheRule ) .around( storageEngineRule ); + private static final Function,StoreType> assertIsPresentAndGet = optional -> + { + assert optional.isPresent() : "Expected optional to be present"; + return optional.get(); + }; + @Test( timeout = 30_000 ) public void shutdownRecordStorageEngineAfterFailedTransaction() throws Throwable { @@ -167,42 +172,19 @@ public void flushAndForce( IOLimiter limiter ) throws IOException } @Test - public void shouldListAllFiles() throws Throwable + public void shouldListAllStoreFiles() throws Throwable { RecordStorageEngine engine = buildRecordStorageEngine(); final Collection files = engine.listStorageFiles(); - assertTrue( files.size() > 0 ); - files.forEach( this::verifyMeta ); - } + Set expectedStoreTypes = Arrays.stream( StoreType.values() ).collect( Collectors.toSet() ); - private void verifyMeta( StoreFileMetadata meta ) - { - final Optional optional = meta.storeType(); - if ( optional.isPresent() ) - { - final StoreType type = optional.get(); - final File file = meta.file(); - final String fileName = file.getName(); - if ( type == StoreType.COUNTS ) - { - final String left = StoreFile.COUNTS_STORE_LEFT.fileName( StoreFileType.STORE ); - final String right = StoreFile.COUNTS_STORE_RIGHT.fileName( StoreFileType.STORE ); - assertThat( fileName, anyOf( equalTo( left ), equalTo( right ) ) ); - } - else - { - final String expected = type.getStoreFile().fileName( StoreFileType.STORE ); - assertThat( fileName, equalTo( expected ) ); - assertTrue( "File does not exist " + file.getAbsolutePath(), fsRule.get().fileExists( file ) ); - } - final int recordSize = meta.recordSize(); - assertTrue( recordSize == RecordFormat.NO_RECORD_SIZE || recordSize > 0 ); - } - else - { - fail( "Assumed all files to have a store type" ); - } + Set actualStoreTypes = files.stream() + .map( storeFileMetadata -> StoreType.typeOf( storeFileMetadata.file().getName() ) ) + .map( assertIsPresentAndGet ) + .collect( Collectors.toSet() ); + + assertEquals( expectedStoreTypes, actualStoreTypes ); } private RecordStorageEngine buildRecordStorageEngine() throws Throwable diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/NeoStoresTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/NeoStoresTest.java index 2b1ac00d95d9c..d324876ca0108 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/NeoStoresTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/NeoStoresTest.java @@ -29,6 +29,7 @@ import java.io.IOException; import java.io.InterruptedIOException; import java.nio.ByteBuffer; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.Iterator; @@ -104,6 +105,7 @@ import static org.neo4j.helpers.collection.MapUtil.stringMap; import static org.neo4j.kernel.api.properties.DefinedProperty.NO_SUCH_PROPERTY; import static org.neo4j.kernel.api.security.SecurityContext.AUTH_DISABLED; +import static org.neo4j.kernel.configuration.Config.embeddedDefaults; import static org.neo4j.kernel.impl.store.RecordStore.getRecord; import static org.neo4j.kernel.impl.store.format.standard.MetaDataRecordFormat.FIELD_NOT_PRESENT; import static org.neo4j.kernel.impl.store.record.RecordLoad.NORMAL; @@ -139,7 +141,7 @@ public class NeoStoresTest public void setUpNeoStores() throws Exception { storeDir = dir.graphDbDir(); - Config config = Config.embeddedDefaults(); + Config config = embeddedDefaults(); pageCache = pageCacheRule.getPageCache( fs.get() ); StoreFactory sf = new StoreFactory( storeDir, config, new DefaultIdGeneratorFactory( fs.get() ), pageCache, fs.get(), NullLogProvider.getInstance() ); @@ -149,7 +151,7 @@ public void setUpNeoStores() throws Exception @Test public void impossibleToGetStoreFromClosedNeoStoresContainer() { - Config config = Config.embeddedDefaults(); + Config config = embeddedDefaults(); StoreFactory sf = new StoreFactory( storeDir, config, new DefaultIdGeneratorFactory( fs.get() ), pageCache, fs.get(), NullLogProvider.getInstance() ); NeoStores neoStores = sf.openAllNeoStores( true ); @@ -166,7 +168,7 @@ public void impossibleToGetStoreFromClosedNeoStoresContainer() @Test public void notAllowCreateDynamicStoreWithNegativeBlockSize() { - Config config = Config.embeddedDefaults(); + Config config = embeddedDefaults(); StoreFactory sf = new StoreFactory( storeDir, config, new DefaultIdGeneratorFactory( fs.get() ), pageCache, fs.get(), NullLogProvider.getInstance() ); @@ -182,7 +184,7 @@ public void notAllowCreateDynamicStoreWithNegativeBlockSize() @Test public void impossibleToGetNotRequestedStore() { - Config config = Config.embeddedDefaults(); + Config config = embeddedDefaults(); StoreFactory sf = new StoreFactory( storeDir, config, new DefaultIdGeneratorFactory( fs.get() ), pageCache, fs.get(), NullLogProvider.getInstance() ); NeoStores neoStores = sf.openNeoStores( true, StoreType.NODE_LABEL ); @@ -541,7 +543,7 @@ public void setVersion() throws Exception assertEquals( 10, MetaDataStore.setRecord( pageCache, new File( storeDir, MetaDataStore.DEFAULT_NAME ).getAbsoluteFile(), Position.LOG_VERSION, 12 ) ); - Config config = Config.embeddedDefaults(); + Config config = embeddedDefaults(); StoreFactory sf = new StoreFactory( storeDir, config, new DefaultIdGeneratorFactory( fileSystem ), pageCache, fileSystem, NullLogProvider.getInstance() ); @@ -598,7 +600,7 @@ public void shouldNotReadNonRecordDataAsRecord() throws Exception public void testSetLatestConstraintTx() throws Exception { // given - Config config = Config.embeddedDefaults(); + Config config = embeddedDefaults(); StoreFactory sf = new StoreFactory( dir.directory(), config, new DefaultIdGeneratorFactory( fs.get() ), pageCacheRule.getPageCache( fs.get() ), fs.get(), NullLogProvider.getInstance() ); @@ -772,7 +774,7 @@ public void shouldCloseAllTheStoreEvenIfExceptionsAreThrown() throws Exception { // given FileSystemAbstraction fileSystem = fs.get(); - Config defaults = Config.embeddedDefaults( singletonMap( counts_store_rotation_timeout.name(), "60m" ) ); + Config defaults = embeddedDefaults( singletonMap( counts_store_rotation_timeout.name(), "60m" ) ); StoreFactory factory = new StoreFactory( storeDir, defaults, new DefaultIdGeneratorFactory( fileSystem ), pageCache, fileSystem, NullLogProvider.getInstance() ); @@ -825,6 +827,44 @@ public void shouldCloseAllTheStoreEvenIfExceptionsAreThrown() throws Exception assertNull( exRef.get() ); } + @Test + public void isPresentAfterCreatingAllStores() throws Exception + { + // given + FileSystemAbstraction fileSystem = fs.get(); + fileSystem.deleteRecursively( storeDir ); + DefaultIdGeneratorFactory idFactory = new DefaultIdGeneratorFactory( fileSystem ); + StoreFactory factory = new StoreFactory( storeDir, embeddedDefaults(), idFactory, pageCache, fileSystem, + NullLogProvider.getInstance() ); + + // when + try ( NeoStores ignore = factory.openAllNeoStores( true ) ) + { + // then + assertTrue( NeoStores.isStorePresent( pageCache, storeDir ) ); + } + } + + @Test + public void isPresentFalseAfterCreatingAllButLastStoreType() throws Exception + { + // given + FileSystemAbstraction fileSystem = fs.get(); + fileSystem.deleteRecursively( storeDir ); + DefaultIdGeneratorFactory idFactory = new DefaultIdGeneratorFactory( fileSystem ); + StoreFactory factory = new StoreFactory( storeDir, embeddedDefaults(), idFactory, pageCache, fileSystem, + NullLogProvider.getInstance() ); + StoreType[] allStoreTypes = StoreType.values(); + StoreType[] allButLastStoreTypes = Arrays.copyOf( allStoreTypes, allStoreTypes.length - 1 ); + + // when + try ( NeoStores ignore = factory.openNeoStores( true, allButLastStoreTypes ) ) + { + // then + assertFalse( NeoStores.isStorePresent( pageCache, storeDir ) ); + } + } + private static class MyPropertyKeyToken extends Token { private static Map stringToIndex = new HashMap<>(); diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/state/NeoStoreFileListingTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/state/NeoStoreFileListingTest.java index 1746c28b29a14..62570d67f492d 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/state/NeoStoreFileListingTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/state/NeoStoreFileListingTest.java @@ -19,7 +19,6 @@ */ package org.neo4j.kernel.impl.transaction.state; -import org.junit.Assert; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -29,27 +28,24 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; -import java.util.HashSet; import java.util.List; -import java.util.Set; +import java.util.Optional; +import java.util.function.Function; +import java.util.stream.Collectors; import org.neo4j.graphdb.ResourceIterator; -import org.neo4j.helpers.collection.Iterables; import org.neo4j.kernel.NeoStoreDataSource; import org.neo4j.kernel.api.labelscan.LabelScanStore; import org.neo4j.kernel.impl.api.LegacyIndexProviderLookup; import org.neo4j.kernel.impl.api.index.IndexingService; -import org.neo4j.kernel.impl.index.IndexConfigStore; -import org.neo4j.kernel.impl.store.MetaDataStore; -import org.neo4j.kernel.impl.store.StoreFile; import org.neo4j.kernel.impl.store.StoreType; -import org.neo4j.kernel.impl.storemigration.StoreFileType; +import org.neo4j.kernel.impl.storemigration.LogFiles; import org.neo4j.kernel.impl.transaction.log.PhysicalLogFile; import org.neo4j.storageengine.api.StorageEngine; import org.neo4j.storageengine.api.StoreFileMetadata; import org.neo4j.test.rule.EmbeddedDatabaseRule; -import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; @@ -63,12 +59,14 @@ public class NeoStoreFileListingTest public EmbeddedDatabaseRule db = new EmbeddedDatabaseRule( getClass() ); private NeoStoreDataSource neoStoreDataSource; private static final String[] STANDARD_STORE_DIR_FILES = new String[]{ + "index", "lock", "debug.log", "neostore", "neostore.id", "neostore.counts.db.a", "neostore.counts.db.b", + "neostore.labelscanstore.db", "neostore.labeltokenstore.db", "neostore.labeltokenstore.db.id", "neostore.labeltokenstore.db.names", @@ -87,6 +85,8 @@ public class NeoStoreFileListingTest "neostore.propertystore.db.index.keys.id", "neostore.propertystore.db.strings", "neostore.propertystore.db.strings.id", + "neostore.relationshipgroupstore.db", + "neostore.relationshipgroupstore.db.id", "neostore.relationshipstore.db", "neostore.relationshipstore.db.id", "neostore.relationshiptypestore.db", @@ -102,6 +102,8 @@ public class NeoStoreFileListingTest "store_lock"}; private static final String[] STANDARD_STORE_DIR_DIRECTORIES = new String[]{"schema", "index", "branched"}; + private static final Function> toStoreType = + fileMetaData -> StoreType.typeOf( fileMetaData.file().getName() ); @Before public void setUp() throws IOException @@ -120,54 +122,6 @@ private void createIndexDbFile() throws IOException } } - private Set expectedStoreFiles( boolean includeLogFiles ) - { - Set storeFileNames = new HashSet<>(); - for ( StoreType type : StoreType.values() ) - { - if ( !type.equals( StoreType.COUNTS ) ) - { - storeFileNames.add( type.getStoreFile().fileName( StoreFileType.STORE ) ); - } - } - if ( includeLogFiles ) - { - storeFileNames.add( PhysicalLogFile.DEFAULT_NAME + ".0" ); - } - storeFileNames.add( IndexConfigStore.INDEX_DB_FILE_NAME ); - return storeFileNames; - } - - private List countStoreFiles() - { - List countStoreFiles = new ArrayList<>(); - countStoreFiles.add( StoreFile.COUNTS_STORE_LEFT.fileName( StoreFileType.STORE ) ); - countStoreFiles.add( StoreFile.COUNTS_STORE_RIGHT.fileName( StoreFileType.STORE ) ); - return countStoreFiles; - } - - @Test - public void shouldNotIncludeTransactionLogFile() throws Exception - { - final ResourceIterator storeFiles = neoStoreDataSource.listStoreFiles( false ); - final Set actual = asSetOfPaths( storeFiles ); - final Set expectedStoreFiles = expectedStoreFiles( false ); - final List countStoreFiles = countStoreFiles(); - assertTrue( diffSet( actual, expectedStoreFiles ), actual.containsAll( expectedStoreFiles ) ); - assertFalse( Collections.disjoint( actual, countStoreFiles ) ); - } - - @Test - public void shouldIncludeTransactionLogFile() throws Exception - { - final ResourceIterator storeFiles = neoStoreDataSource.listStoreFiles( true ); - final Set actual = asSetOfPaths( storeFiles ); - final Set expectedStoreFiles = expectedStoreFiles( true ); - final List countStoreFiles = countStoreFiles(); - assertTrue( diffSet( actual, expectedStoreFiles ), actual.containsAll( expectedStoreFiles ) ); - assertFalse( Collections.disjoint( actual, countStoreFiles ) ); - } - @Test public void shouldCloseIndexAndLabelScanSnapshots() throws Exception { @@ -197,64 +151,65 @@ public void shouldCloseIndexAndLabelScanSnapshots() throws Exception } @Test - public void shouldListNeostoreDbLast() throws Exception + public void shouldListMetaDataStoreLast() throws Exception { - Boolean[] foundStoreType = new Boolean[StoreType.values().length]; - boolean foundTxLogs = false; - - final ResourceIterator storeFiles = - neoStoreDataSource.listStoreFiles( true ); + assertTrue( neoStoreDataSource.listStoreFiles( false ).stream() + .reduce( ( a, b ) -> b ) + .map( toStoreType ) + .filter( Optional::isPresent ) + .map( Optional::get ) + .filter( StoreType.META_DATA::equals ) + .isPresent() ); + } - while ( storeFiles.hasNext() ) - { - final StoreFileMetadata storeFile = storeFiles.next(); - if ( storeFile.storeType().isPresent() ) - { - StoreType storeType = storeFile.storeType().get(); - foundStoreType[storeType.ordinal()] = true; - if ( storeType == StoreType.META_DATA ) - { - Arrays.stream( foundStoreType ).forEach( Assert::assertTrue ); - assertTrue( "Transaction logs was not listed before neostore.db", foundTxLogs ); - } - } - else if ( transactionLogFile( storeFile.file().getName() ) ) - { - foundTxLogs = true; - } - } + @Test + public void shouldListMetaDataStoreLastWithTxLogs() throws Exception + { + assertTrue( neoStoreDataSource.listStoreFiles( true ).stream() + .reduce( ( a, b ) -> b ) + .map( toStoreType ) + .filter( Optional::isPresent ) + .map( Optional::get ) + .filter( StoreType.META_DATA::equals ) + .isPresent() ); } - private boolean transactionLogFile( String name ) + @Test + public void shouldListTxLogFiles() throws Exception { - return name.startsWith( MetaDataStore.DEFAULT_NAME + ".transaction" ) && !name.endsWith( ".active" ); + assertTrue( neoStoreDataSource.listStoreFiles( true ).stream() + .map( metaData -> metaData.file().getName() ) + .anyMatch( fileName -> LogFiles.FILENAME_FILTER.accept( null, fileName ) ) ); } - private void filesInStoreDirAre( File storeDir, String[] filenames, String[] dirs ) + @Test + public void shouldNotListTxLogFiles() throws Exception { - ArrayList files = new ArrayList<>(); - mockFiles( filenames, files, false ); - mockFiles( dirs, files, true ); - when( storeDir.listFiles() ).thenReturn( files.toArray( new File[files.size()] ) ); + assertTrue( neoStoreDataSource.listStoreFiles( false ).stream() + .map( metaData -> metaData.file().getName() ) + .noneMatch( fileName -> LogFiles.FILENAME_FILTER.accept( null, fileName ) ) ); } - private String diffSet( Set actual, Set expected ) + @Test + public void shouldListNeostoreFiles() throws Exception { - Set extra = new HashSet<>( actual ); - Set missing = new HashSet<>( expected ); - extra.removeAll( expected ); - missing.removeAll( actual ); - return "Extra entries: " + extra + "\nMissing entries: " + missing; + StoreType[] values = StoreType.values(); + ResourceIterator storeFiles = + neoStoreDataSource.listStoreFiles( false ); + List listedStoreFiles = storeFiles.stream() + .map( toStoreType ) + .filter( Optional::isPresent ) + .map( Optional::get ) + .collect( Collectors.toList() ); + assertEquals( Arrays.asList(values), listedStoreFiles ); } - private Set asSetOfPaths( ResourceIterator result ) + private void filesInStoreDirAre( File storeDir, String[] filenames, String[] dirs ) { - List names = new ArrayList<>(); - while ( result.hasNext() ) - { - names.add( result.next().file().getName() ); - } - return Iterables.asUniqueSet( names ); + ArrayList files = new ArrayList<>(); + mockFiles( filenames, files, false ); + mockFiles( dirs, files, true ); + when( storeDir.listFiles() ).thenReturn( files.toArray( new File[files.size()] ) ); } private ResourceIterator scanStoreFilesAre( LabelScanStore labelScanStore, String[] fileNames ) diff --git a/community/udc/src/test/java/org/neo4j/ext/udc/impl/DefaultUdcInformationCollectorTest.java b/community/udc/src/test/java/org/neo4j/ext/udc/impl/DefaultUdcInformationCollectorTest.java index d755465f67cfb..60d6aa1cd3365 100644 --- a/community/udc/src/test/java/org/neo4j/ext/udc/impl/DefaultUdcInformationCollectorTest.java +++ b/community/udc/src/test/java/org/neo4j/ext/udc/impl/DefaultUdcInformationCollectorTest.java @@ -29,7 +29,6 @@ import java.util.HashMap; import java.util.HashSet; import java.util.Map; -import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -201,7 +200,7 @@ private Set testFiles() throws Exception private Set toMeta( File... files ) { return Arrays.stream( files ) - .map( file -> new StoreFileMetadata( file, Optional.empty(), RecordFormat.NO_RECORD_SIZE ) ) + .map( file -> new StoreFileMetadata( file, RecordFormat.NO_RECORD_SIZE ) ) .collect( Collectors.toCollection( HashSet::new ) ); } diff --git a/integrationtests/src/test/java/org/neo4j/ha/HAClusterStartupIT.java b/integrationtests/src/test/java/org/neo4j/ha/HAClusterStartupIT.java index 000cc5d18cf6b..7abb8a0f7106c 100644 --- a/integrationtests/src/test/java/org/neo4j/ha/HAClusterStartupIT.java +++ b/integrationtests/src/test/java/org/neo4j/ha/HAClusterStartupIT.java @@ -241,7 +241,7 @@ private static void deleteAllLogsOn( HighlyAvailableGraphDatabase instance ) private static void deleteAllLogsOn( File storeDirectory ) { - File[] files = storeDirectory.listFiles( new LogFiles.LogicalLogFilenameFilter() ); + File[] files = storeDirectory.listFiles( LogFiles.FILENAME_FILTER ); assertNotNull( files ); for ( File file : files ) {