diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigrator.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigrator.java index acc9a3a0ab5d2..5ac486d4e476a 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigrator.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigrator.java @@ -27,6 +27,7 @@ import java.io.Reader; import java.io.Writer; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.List; import java.util.Random; @@ -92,7 +93,6 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.Arrays.asList; -import static org.neo4j.helpers.collection.Iterables.iterable; import static org.neo4j.kernel.impl.store.MetaDataStore.DEFAULT_NAME; import static org.neo4j.kernel.impl.store.format.Capability.VERSION_TRAILERS; import static org.neo4j.kernel.impl.store.format.InternalRecordFormatSelector.fromVersion; @@ -419,10 +419,17 @@ private void prepareBatchImportMigration( File storeDir, File migrationDir, Reco // that dynamic record store over before doing the "batch import". // Copying this file just as-is assumes that the format hasn't change. If that happens we're in // a different situation, where we first need to migrate this file. + + // The token stores also need to be migrated because we use those as-is and ask for their high ids + // when using the importer in the store migration scenario. + StoreFile[] storesFilesToMigrate = { + StoreFile.LABEL_TOKEN_STORE, StoreFile.LABEL_TOKEN_NAMES_STORE, + StoreFile.PROPERTY_KEY_TOKEN_STORE, StoreFile.PROPERTY_KEY_TOKEN_NAMES_STORE, + StoreFile.RELATIONSHIP_TYPE_TOKEN_STORE, StoreFile.RELATIONSHIP_TYPE_TOKEN_NAMES_STORE, + StoreFile.NODE_LABEL_STORE}; if ( newFormat.dynamic().equals( oldFormat.dynamic() ) ) { - Iterable storeFiles = iterable( StoreFile.NODE_LABEL_STORE ); - StoreFile.fileOperation( COPY, fileSystem, storeDir, migrationDir, storeFiles, + StoreFile.fileOperation( COPY, fileSystem, storeDir, migrationDir, Arrays.asList(storesFilesToMigrate), true, // OK if it's not there (1.9) ExistingTargetStrategy.FAIL, StoreFileType.values() ); } diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/MetaDataStoreTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/MetaDataStoreTest.java index 41d19cb04790c..19b95cfb033cd 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/MetaDataStoreTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/MetaDataStoreTest.java @@ -56,6 +56,8 @@ import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import static org.neo4j.kernel.impl.store.MetaDataStore.versionStringToLong; +import static org.neo4j.kernel.impl.store.format.InternalRecordFormatSelector.select; public class MetaDataStoreTest { @@ -581,10 +583,8 @@ public void mustSupportScanningAllRecords() throws Exception fs.mkdir( STORE_DIR ); fs.create( file ).close(); MetaDataStore.Position[] positions = MetaDataStore.Position.values(); - for ( MetaDataStore.Position position : positions ) - { - MetaDataStore.setRecord( pageCache, file, position, position.ordinal() + 1 ); - } + long storeVersion = versionStringToLong( select().storeVersion()); + writeCorrectMetaDataRecord( file, positions, storeVersion ); List actualValues = new ArrayList<>(); try ( MetaDataStore store = newMetaDataStore() ) @@ -595,8 +595,16 @@ public void mustSupportScanningAllRecords() throws Exception } ); } - List expectedValues = Arrays.stream( positions ).map( - p -> p.ordinal() + 1L ).collect( Collectors.toList() ); + List expectedValues = Arrays.stream( positions ).map( p -> { + if ( p == MetaDataStore.Position.STORE_VERSION ) + { + return storeVersion; + } + else + { + return p.ordinal() + 1L; + } + } ).collect( Collectors.toList() ); assertThat( actualValues, is( expectedValues ) ); } @@ -608,10 +616,8 @@ public void mustSupportScanningAllRecordsWithRecordCursor() throws Exception fs.mkdir( STORE_DIR ); fs.create( file ).close(); MetaDataStore.Position[] positions = MetaDataStore.Position.values(); - for ( MetaDataStore.Position position : positions ) - { - MetaDataStore.setRecord( pageCache, file, position, position.ordinal() + 1 ); - } + long storeVersion = versionStringToLong( select().storeVersion()); + writeCorrectMetaDataRecord( file, positions, storeVersion ); List actualValues = new ArrayList<>(); try ( MetaDataStore store = newMetaDataStore() ) @@ -631,9 +637,34 @@ public void mustSupportScanningAllRecordsWithRecordCursor() throws Exception } } - List expectedValues = Arrays.stream( positions ).map( - p -> p.ordinal() + 1L ).collect( Collectors.toList() ); + List expectedValues = Arrays.stream( positions ).map( p -> { + if ( p == MetaDataStore.Position.STORE_VERSION ) + { + return storeVersion; + } + else + { + return p.ordinal() + 1L; + } + } ).collect( Collectors.toList() ); + assertThat( actualValues, is( expectedValues ) ); } + + private void writeCorrectMetaDataRecord( File file, MetaDataStore.Position[] positions, long storeVersion ) + throws IOException + { + for ( MetaDataStore.Position position : positions ) + { + if ( position == MetaDataStore.Position.STORE_VERSION ) + { + MetaDataStore.setRecord( pageCache, file, position, storeVersion ); + } + else + { + MetaDataStore.setRecord( pageCache, file, position, position.ordinal() + 1 ); + } + } + } } diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/NeoStoresTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/NeoStoresTest.java index eee340d8f95d2..829e6c3fd51f4 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/NeoStoresTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/NeoStoresTest.java @@ -61,6 +61,7 @@ import org.neo4j.kernel.impl.core.RelationshipTypeToken; import org.neo4j.kernel.impl.storageengine.impl.recordstorage.RecordStorageEngine; import org.neo4j.kernel.impl.store.MetaDataStore.Position; +import org.neo4j.kernel.impl.store.format.InternalRecordFormatSelector; import org.neo4j.kernel.impl.store.format.lowlimit.DynamicRecordFormat; import org.neo4j.kernel.impl.store.id.DefaultIdGeneratorFactory; import org.neo4j.kernel.impl.store.record.PropertyBlock; @@ -92,6 +93,7 @@ import static org.junit.Assert.assertTrue; import static org.neo4j.helpers.collection.MapUtil.stringMap; import static org.neo4j.kernel.impl.store.RecordStore.getRecord; +import static org.neo4j.kernel.impl.store.format.InternalRecordFormatSelector.select; import static org.neo4j.kernel.impl.store.format.lowlimit.MetaDataRecordFormat.FIELD_NOT_PRESENT; import static org.neo4j.kernel.impl.store.record.RecordLoad.NORMAL; @@ -505,7 +507,7 @@ public void shouldNotReadNonRecordDataAsRecord() throws Exception FileSystemAbstraction fileSystem = fs.get(); File neoStoreDir = new File( "/tmp/graph.db/neostore" ).getAbsoluteFile(); StoreFactory factory = new StoreFactory( fileSystem, neoStoreDir, pageCache, NullLogProvider.getInstance() ); - + long recordVersion = MetaDataStore.versionStringToLong( InternalRecordFormatSelector.select().storeVersion() ); try ( NeoStores neoStores = factory.openAllNeoStores( true ) ) { MetaDataStore metaDataStore = neoStores.getMetaDataStore(); @@ -513,7 +515,7 @@ public void shouldNotReadNonRecordDataAsRecord() throws Exception metaDataStore.setRandomNumber( 4 ); metaDataStore.setCurrentLogVersion( 5 ); metaDataStore.setLastCommittedAndClosedTransactionId( 6, 0, 0, 0 ); - metaDataStore.setStoreVersion( 7 ); + metaDataStore.setStoreVersion( recordVersion ); metaDataStore.setGraphNextProp( 8 ); metaDataStore.setLatestConstraintIntroducingTx( 9 ); } @@ -523,8 +525,11 @@ public void shouldNotReadNonRecordDataAsRecord() throws Exception { channel.position( 0 ); channel.write( ByteBuffer.wrap( UTF8.encode( "This is some data that is not a record." ) ) ); + } + MetaDataStore.setRecord( pageCache, file, Position.STORE_VERSION, recordVersion ); + try ( NeoStores neoStores = factory.openAllNeoStores() ) { MetaDataStore metaDataStore = neoStores.getMetaDataStore(); @@ -532,7 +537,7 @@ public void shouldNotReadNonRecordDataAsRecord() throws Exception assertEquals( FIELD_NOT_PRESENT, metaDataStore.getRandomNumber() ); assertEquals( FIELD_NOT_PRESENT, metaDataStore.getCurrentLogVersion() ); assertEquals( FIELD_NOT_PRESENT, metaDataStore.getLastCommittedTransactionId() ); - assertEquals( FIELD_NOT_PRESENT, metaDataStore.getStoreVersion() ); + assertEquals( recordVersion, metaDataStore.getStoreVersion() ); assertEquals( 8, metaDataStore.getGraphNextProp() ); assertEquals( 9, metaDataStore.getLatestConstraintIntroducingTx() ); } @@ -614,7 +619,7 @@ public void shouldAddUpgradeFieldsToTheNeoStoreIfNotPresent() throws IOException FileSystemAbstraction fileSystem = fs.get(); File neoStoreDir = new File( "/tmp/graph.db/neostore" ).getAbsoluteFile(); StoreFactory factory = new StoreFactory( fileSystem, neoStoreDir, pageCache, NullLogProvider.getInstance() ); - + long recordVersion = MetaDataStore.versionStringToLong( select().storeVersion() ); try ( NeoStores neoStores = factory.openAllNeoStores( true ) ) { MetaDataStore metaDataStore = neoStores.getMetaDataStore(); @@ -622,7 +627,7 @@ public void shouldAddUpgradeFieldsToTheNeoStoreIfNotPresent() throws IOException metaDataStore.setRandomNumber( 4 ); metaDataStore.setCurrentLogVersion( 5 ); metaDataStore.setLastCommittedAndClosedTransactionId( 6, 0, 0, 0 ); - metaDataStore.setStoreVersion( 7 ); + metaDataStore.setStoreVersion( recordVersion ); metaDataStore.setGraphNextProp( 8 ); metaDataStore.setLatestConstraintIntroducingTx( 9 ); } @@ -643,7 +648,7 @@ public void shouldAddUpgradeFieldsToTheNeoStoreIfNotPresent() throws IOException assertEquals( 4, metaDataStore.getRandomNumber() ); assertEquals( 5, metaDataStore.getCurrentLogVersion() ); assertEquals( 6, metaDataStore.getLastCommittedTransactionId() ); - assertEquals( 7, metaDataStore.getStoreVersion() ); + assertEquals( recordVersion, metaDataStore.getStoreVersion() ); assertEquals( 8, metaDataStore.getGraphNextProp() ); assertEquals( 9, metaDataStore.getLatestConstraintIntroducingTx() ); assertEquals( new TransactionId( 10, 11 ), metaDataStore.getUpgradeTransaction() ); diff --git a/community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/store/BatchingTokenRepositoryTest.java b/community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/store/BatchingTokenRepositoryTest.java index 8d2f7f432f0d0..49081ffd578da 100644 --- a/community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/store/BatchingTokenRepositoryTest.java +++ b/community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/store/BatchingTokenRepositoryTest.java @@ -22,10 +22,12 @@ import org.junit.Test; import org.neo4j.kernel.impl.store.NodeLabelsField; +import org.neo4j.kernel.impl.store.TokenStore; import org.neo4j.unsafe.impl.batchimport.store.BatchingTokenRepository.BatchingLabelTokenRepository; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; public class BatchingTokenRepositoryTest { @@ -33,7 +35,8 @@ public class BatchingTokenRepositoryTest public void shouldDedupLabelIds() throws Exception { // GIVEN - BatchingLabelTokenRepository repo = new BatchingLabelTokenRepository( null ); + @SuppressWarnings( "unchecked" ) + BatchingLabelTokenRepository repo = new BatchingLabelTokenRepository( mock( TokenStore.class ) ); // WHEN long[] ids = repo.getOrCreateIds( new String[] {"One", "Two", "One"} ); @@ -46,7 +49,8 @@ public void shouldDedupLabelIds() throws Exception public void shouldSortLabelIds() throws Exception { // GIVEN - BatchingLabelTokenRepository repo = new BatchingLabelTokenRepository( null ); + @SuppressWarnings( "unchecked" ) + BatchingLabelTokenRepository repo = new BatchingLabelTokenRepository( mock( TokenStore.class ) ); long[] expected = new long[] { repo.getOrCreateId( "One" ), repo.getOrCreateId( "Two" ),