diff --git a/community/consistency-check/src/test/java/org/neo4j/unsafe/impl/batchimport/ParallelBatchImporterTest.java b/community/consistency-check/src/test/java/org/neo4j/unsafe/impl/batchimport/ParallelBatchImporterTest.java index ff9509618860f..7f1c7420a4e17 100644 --- a/community/consistency-check/src/test/java/org/neo4j/unsafe/impl/batchimport/ParallelBatchImporterTest.java +++ b/community/consistency-check/src/test/java/org/neo4j/unsafe/impl/batchimport/ParallelBatchImporterTest.java @@ -52,11 +52,11 @@ import org.neo4j.graphdb.RelationshipType; import org.neo4j.graphdb.Transaction; import org.neo4j.graphdb.factory.GraphDatabaseSettings; -import org.neo4j.helpers.collection.MapUtil; import org.neo4j.helpers.progress.ProgressMonitorFactory; import org.neo4j.io.fs.DefaultFileSystemAbstraction; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.logging.NullLogService; +import org.neo4j.kernel.impl.store.format.RecordFormats; import org.neo4j.kernel.impl.store.format.standard.StandardV3_0; import org.neo4j.logging.NullLogProvider; import org.neo4j.test.Randoms; @@ -162,7 +162,7 @@ public void shouldImportCsvData() throws Exception ExecutionMonitor processorAssigner = eagerRandomSaturation( config.maxNumberOfProcessors() ); final BatchImporter inserter = new ParallelBatchImporter( directory.graphDbDir(), new DefaultFileSystemAbstraction(), config, NullLogService.getInstance(), - processorAssigner, EMPTY, new Config( MapUtil.stringMap( GraphDatabaseSettings.record_format.name(), getFormatName() ) ) ); + processorAssigner, EMPTY, Config.empty(), getFormat() ); boolean successful = false; IdGroupDistribution groups = new IdGroupDistribution( NODE_COUNT, 5, random.random() ); @@ -234,9 +234,9 @@ private void assertConsistent( File storeDir ) throws ConsistencyCheckIncomplete result.isSuccessful() ); } - protected String getFormatName() + protected RecordFormats getFormat() { - return StandardV3_0.NAME; + return StandardV3_0.RECORD_FORMATS; } public abstract static class InputIdGenerator diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/format/Capability.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/format/Capability.java index 4af806f61b918..4829e3b6a9075 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/format/Capability.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/format/Capability.java @@ -37,6 +37,11 @@ public enum Capability */ DENSE_NODES( CapabilityType.FORMAT, CapabilityType.STORE ), + /** + * 3 bytes relationship type support + */ + RELATIONSHIP_TYPE_3BYTES( CapabilityType.FORMAT, CapabilityType.STORE ), + /** * Store has version trailers in the end of cleanly shut down store */ diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigrator.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigrator.java index 12110e157d297..f5ef10aae8224 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigrator.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigrator.java @@ -416,7 +416,7 @@ private void migrateWithBatchImporter( File storeDir, File migrationDir, long la BatchImporter importer = new ParallelBatchImporter( migrationDir.getAbsoluteFile(), fileSystem, importConfig, logService, withDynamicProcessorAssignment( migrationBatchImporterMonitor( legacyStore, progressMonitor, - importConfig ), importConfig ), additionalInitialIds, config ); + importConfig ), importConfig ), additionalInitialIds, config, newFormat ); InputIterable nodes = legacyNodesAsInput( legacyStore, requiresPropertyMigration, nodeInputCursors ); InputIterable relationships = diff --git a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/ParallelBatchImporter.java b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/ParallelBatchImporter.java index 857fef5a6a275..74c1fafae51f0 100644 --- a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/ParallelBatchImporter.java +++ b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/ParallelBatchImporter.java @@ -58,7 +58,6 @@ import static java.lang.Math.max; import static java.lang.String.format; import static java.lang.System.currentTimeMillis; - import static org.neo4j.helpers.Format.bytes; import static org.neo4j.helpers.collection.Iterators.asSet; import static org.neo4j.io.ByteUnit.mebiBytes; @@ -90,6 +89,7 @@ public class ParallelBatchImporter implements BatchImporter private final ExecutionMonitor executionMonitor; private final AdditionalInitialIds additionalInitialIds; private final Config dbConfig; + private final RecordFormats recordFormats; /** * Advanced usage of the parallel batch importer, for special and very specific cases. Please use @@ -98,13 +98,14 @@ public class ParallelBatchImporter implements BatchImporter public ParallelBatchImporter( File storeDir, FileSystemAbstraction fileSystem, Configuration config, LogService logService, ExecutionMonitor executionMonitor, AdditionalInitialIds additionalInitialIds, - Config dbConfig ) + Config dbConfig, RecordFormats recordFormats ) { this.storeDir = storeDir; this.fileSystem = fileSystem; this.config = config; this.logService = logService; this.dbConfig = dbConfig; + this.recordFormats = recordFormats; this.log = logService.getInternalLogProvider().getLog( getClass() ); this.executionMonitor = executionMonitor; this.additionalInitialIds = additionalInitialIds; @@ -119,7 +120,8 @@ public ParallelBatchImporter( File storeDir, Configuration config, LogService lo ExecutionMonitor executionMonitor, Config dbConfig ) { this( storeDir, new DefaultFileSystemAbstraction(), config, logService, - withDynamicProcessorAssignment( executionMonitor, config ), EMPTY, dbConfig ); + withDynamicProcessorAssignment( executionMonitor, config ), EMPTY, dbConfig, + RecordFormatSelector.selectForConfig( dbConfig, NullLogProvider.getInstance() )); } @Override @@ -135,7 +137,6 @@ public void doImport( Input input ) throws IOException boolean hasBadEntries = false; File badFile = new File( storeDir, Configuration.BAD_FILE_NAME ); CountingStoreUpdateMonitor storeUpdateMonitor = new CountingStoreUpdateMonitor(); - RecordFormats recordFormats = RecordFormatSelector.selectForConfig( dbConfig, NullLogProvider.getInstance() ); try ( BatchingNeoStores neoStore = new BatchingNeoStores( fileSystem, storeDir, recordFormats, config, logService, additionalInitialIds, dbConfig ); CountsAccessor.Updater countsUpdater = neoStore.getCountsStore().reset( diff --git a/community/neo4j/src/test/java/schema/MultipleIndexPopulationStressIT.java b/community/neo4j/src/test/java/schema/MultipleIndexPopulationStressIT.java index 45396050b31f4..3e23d5506224b 100644 --- a/community/neo4j/src/test/java/schema/MultipleIndexPopulationStressIT.java +++ b/community/neo4j/src/test/java/schema/MultipleIndexPopulationStressIT.java @@ -52,6 +52,8 @@ import org.neo4j.kernel.impl.api.index.BatchingMultipleIndexPopulator; import org.neo4j.kernel.impl.api.index.MultipleIndexPopulator; import org.neo4j.kernel.impl.logging.NullLogService; +import org.neo4j.kernel.impl.store.format.RecordFormatSelector; +import org.neo4j.kernel.impl.store.format.RecordFormats; import org.neo4j.logging.NullLogProvider; import org.neo4j.test.Randoms; import org.neo4j.test.RepeatRule; @@ -298,8 +300,11 @@ private void changeRandomNode( GraphDatabaseService db, int nodeCount, Randoms r private void createRandomData( int count ) throws IOException { + Config config = Config.empty(); + RecordFormats recordFormats = + RecordFormatSelector.selectForConfig( config, NullLogProvider.getInstance() ); BatchImporter importer = new ParallelBatchImporter( directory.graphDbDir(), fs, - DEFAULT, NullLogService.getInstance(), ExecutionMonitors.invisible(), EMPTY, Config.empty() ); + DEFAULT, NullLogService.getInstance(), ExecutionMonitors.invisible(), EMPTY, config, recordFormats ); importer.doImport( new RandomDataInput( count ) ); } diff --git a/enterprise/kernel/src/main/java/org/neo4j/kernel/impl/store/format/highlimit/HighLimit.java b/enterprise/kernel/src/main/java/org/neo4j/kernel/impl/store/format/highlimit/HighLimit.java index f9344cbdfc981..1ef6643e5e30e 100644 --- a/enterprise/kernel/src/main/java/org/neo4j/kernel/impl/store/format/highlimit/HighLimit.java +++ b/enterprise/kernel/src/main/java/org/neo4j/kernel/impl/store/format/highlimit/HighLimit.java @@ -56,7 +56,8 @@ public class HighLimit extends BaseRecordFormats public HighLimit() { - super( STORE_VERSION, 3, Capability.DENSE_NODES, Capability.SCHEMA, Capability.LUCENE_5 ); + super( STORE_VERSION, 3, Capability.DENSE_NODES, Capability.RELATIONSHIP_TYPE_3BYTES, Capability.SCHEMA, + Capability.LUCENE_5 ); } @Override diff --git a/enterprise/kernel/src/test/java/org/neo4j/kernel/impl/store/format/highlimit/HighLimitStoreMigrationTest.java b/enterprise/kernel/src/test/java/org/neo4j/kernel/impl/store/format/highlimit/HighLimitStoreMigrationTest.java index d7601e6149885..5de41a5639355 100644 --- a/enterprise/kernel/src/test/java/org/neo4j/kernel/impl/store/format/highlimit/HighLimitStoreMigrationTest.java +++ b/enterprise/kernel/src/test/java/org/neo4j/kernel/impl/store/format/highlimit/HighLimitStoreMigrationTest.java @@ -19,17 +19,14 @@ */ package org.neo4j.kernel.impl.store.format.highlimit; -import org.hamcrest.CoreMatchers; import org.junit.Rule; import org.junit.Test; +import org.junit.rules.RuleChain; import java.io.File; import java.io.IOException; -import java.util.Set; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import org.neo4j.graphdb.mockfs.EphemeralFileSystemAbstraction; +import org.neo4j.io.fs.DefaultFileSystemAbstraction; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; import org.neo4j.kernel.api.index.SchemaIndexProvider; @@ -41,57 +38,65 @@ import org.neo4j.kernel.impl.storemigration.monitoring.MigrationProgressMonitor; import org.neo4j.kernel.impl.storemigration.participant.StoreMigrator; import org.neo4j.test.rule.PageCacheRule; +import org.neo4j.test.rule.TestDirectory; +import org.neo4j.test.rule.fs.DefaultFileSystemRule; -import static org.junit.Assert.assertThat; -import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.mockito.Mockito.mock; import static org.neo4j.kernel.impl.store.MetaDataStore.Position.STORE_VERSION; public class HighLimitStoreMigrationTest { + private final PageCacheRule pageCacheRule = new PageCacheRule(); + private final TestDirectory testDirectory = TestDirectory.testDirectory(); + private final DefaultFileSystemRule fileSystemRule = new DefaultFileSystemRule(); @Rule - public final PageCacheRule pageCacheRule = new PageCacheRule(); - private final FileSystemAbstraction fileSystem = new EphemeralFileSystemAbstraction(); + public RuleChain chain = RuleChain.outerRule( pageCacheRule ) + .around( fileSystemRule ) + .around( testDirectory ); @Test - public void haveSameFormatCapabilitiesAsHighLimit3_0() + public void haveDifferentFormatCapabilitiesAsHighLimit3_0() { - assertTrue( HighLimit.RECORD_FORMATS.hasSameCapabilities( HighLimitV3_0_0.RECORD_FORMATS, CapabilityType.FORMAT ) ); + assertFalse( HighLimit.RECORD_FORMATS.hasSameCapabilities( HighLimitV3_0_0.RECORD_FORMATS, CapabilityType.FORMAT ) ); } @Test - public void doNotMigrateHighLimit3_0StoreFiles() throws IOException + public void migrateHighLimit3_0StoreFiles() throws IOException { + DefaultFileSystemAbstraction fileSystem = fileSystemRule.get(); PageCache pageCache = pageCacheRule.getPageCache( fileSystem ); SchemaIndexProvider schemaIndexProvider = mock( SchemaIndexProvider.class ); + StoreMigrator migrator = new StoreMigrator( fileSystem, pageCache, Config.empty(), NullLogService.getInstance(), schemaIndexProvider ); - File storeDir = new File( "storeDir" ); - File migrationDir = new File( "migrationDir" ); + File storeDir = new File( testDirectory.graphDbDir(), "storeDir" ); + File migrationDir = new File( testDirectory.graphDbDir(), "migrationDir" ); fileSystem.mkdir( migrationDir ); + fileSystem.mkdir( storeDir ); - prepareNeoStoreFile( storeDir, HighLimitV3_0_0.STORE_VERSION, pageCache ); + prepareNeoStoreFile( fileSystem, storeDir, HighLimitV3_0_0.STORE_VERSION, pageCache ); MigrationProgressMonitor.Section progressMonitor = mock( MigrationProgressMonitor.Section.class ); + migrator.migrate( storeDir, migrationDir, progressMonitor, HighLimitV3_0_0.STORE_VERSION, HighLimit.STORE_VERSION ); - File[] migrationFiles = fileSystem.listFiles( migrationDir ); - Set fileNames = Stream.of( migrationFiles ).map( File::getName ).collect( Collectors.toSet() ); - assertThat( "Only specified files should be created after migration attempt from 3.0 to 3.1 using high limit " + - "format. Since formats are compatible and migration is not required.", fileNames, - CoreMatchers.hasItems( "lastxinformation", "lastxlogposition" ) ); + int newStoreFilesCount = fileSystem.listFiles( migrationDir ).length; + assertEquals( "Store should be migrated and new store files should be created.", 17, newStoreFilesCount ); } - private File prepareNeoStoreFile( File storeDir, String storeVersion, PageCache pageCache ) throws IOException + private File prepareNeoStoreFile( FileSystemAbstraction fileSystem, File storeDir, String storeVersion, + PageCache pageCache ) throws IOException { - File neoStoreFile = createNeoStoreFile( storeDir ); + File neoStoreFile = createNeoStoreFile( fileSystem, storeDir ); long value = MetaDataStore.versionStringToLong( storeVersion ); MetaDataStore.setRecord( pageCache, neoStoreFile, STORE_VERSION, value ); return neoStoreFile; } - private File createNeoStoreFile( File storeDir ) throws IOException + private File createNeoStoreFile( FileSystemAbstraction fileSystem, File storeDir ) throws IOException { fileSystem.mkdir( storeDir ); File neoStoreFile = new File( storeDir, MetaDataStore.DEFAULT_NAME ); diff --git a/enterprise/neo4j-enterprise/src/test/java/batchimport/ParallelBatchImporterTest.java b/enterprise/neo4j-enterprise/src/test/java/batchimport/ParallelBatchImporterTest.java index f4add5514c4ce..31d010ed1ee48 100644 --- a/enterprise/neo4j-enterprise/src/test/java/batchimport/ParallelBatchImporterTest.java +++ b/enterprise/neo4j-enterprise/src/test/java/batchimport/ParallelBatchImporterTest.java @@ -19,6 +19,7 @@ */ package batchimport; +import org.neo4j.kernel.impl.store.format.RecordFormats; import org.neo4j.kernel.impl.store.format.highlimit.HighLimit; import org.neo4j.unsafe.impl.batchimport.ParallelBatchImporter; import org.neo4j.unsafe.impl.batchimport.cache.idmapping.IdGenerator; @@ -36,8 +37,8 @@ public ParallelBatchImporterTest( InputIdGenerator inputIdGenerator, IdMapper id } @Override - public String getFormatName() + public RecordFormats getFormat() { - return HighLimit.NAME; + return HighLimit.RECORD_FORMATS; } }