Skip to content

Commit

Permalink
Copy token stores before batch migration.
Browse files Browse the repository at this point in the history 
 Without copying token stores the highid would be
 incorrect. Also fixed some wrong test assumptions
 now that a correct store version in metadata store
 is required.
  • Loading branch information
Max Sumrall committed Mar 18, 2016
1 parent aec397c commit 0b192fd
Show file tree
Hide file tree
Showing 4 changed files with 70 additions and 23 deletions.
13 changes: 10 additions & 3 deletions .../kernel/src/main/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigrator.java
View file
Expand Up @@ -27,6 +27,7 @@
import java.io.Reader;
import java.io.Writer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import java.util.Random;
Expand Down Expand Up @@ -92,7 +93,6 @@

import static java.nio.charset.StandardCharsets.UTF_8;
import static java.util.Arrays.asList;
import static org.neo4j.helpers.collection.Iterables.iterable;
import static org.neo4j.kernel.impl.store.MetaDataStore.DEFAULT_NAME;
import static org.neo4j.kernel.impl.store.format.Capability.VERSION_TRAILERS;
import static org.neo4j.kernel.impl.store.format.InternalRecordFormatSelector.fromVersion;
Expand Down Expand Up @@ -419,10 +419,17 @@ private void prepareBatchImportMigration( File storeDir, File migrationDir, Reco
// that dynamic record store over before doing the "batch import".
// Copying this file just as-is assumes that the format hasn't change. If that happens we're in
// a different situation, where we first need to migrate this file.

// The token stores also need to be migrated because we use those as-is and ask for their high ids
// when using the importer in the store migration scenario.
StoreFile[] storesFilesToMigrate = {
StoreFile.LABEL_TOKEN_STORE, StoreFile.LABEL_TOKEN_NAMES_STORE,
StoreFile.PROPERTY_KEY_TOKEN_STORE, StoreFile.PROPERTY_KEY_TOKEN_NAMES_STORE,
StoreFile.RELATIONSHIP_TYPE_TOKEN_STORE, StoreFile.RELATIONSHIP_TYPE_TOKEN_NAMES_STORE,
StoreFile.NODE_LABEL_STORE};
if ( newFormat.dynamic().equals( oldFormat.dynamic() ) )
{
Iterable<StoreFile> storeFiles = iterable( StoreFile.NODE_LABEL_STORE );
StoreFile.fileOperation( COPY, fileSystem, storeDir, migrationDir, storeFiles,
StoreFile.fileOperation( COPY, fileSystem, storeDir, migrationDir, Arrays.asList(storesFilesToMigrate),
true, // OK if it's not there (1.9)
ExistingTargetStrategy.FAIL, StoreFileType.values() );
}
Expand Down
55 changes: 43 additions & 12 deletions community/kernel/src/test/java/org/neo4j/kernel/impl/store/MetaDataStoreTest.java
View file
Expand Up @@ -56,6 +56,8 @@
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.neo4j.kernel.impl.store.MetaDataStore.versionStringToLong;
import static org.neo4j.kernel.impl.store.format.InternalRecordFormatSelector.select;

public class MetaDataStoreTest
{
Expand Down Expand Up @@ -581,10 +583,8 @@ public void mustSupportScanningAllRecords() throws Exception
fs.mkdir( STORE_DIR );
fs.create( file ).close();
MetaDataStore.Position[] positions = MetaDataStore.Position.values();
for ( MetaDataStore.Position position : positions )
{
MetaDataStore.setRecord( pageCache, file, position, position.ordinal() + 1 );
}
long storeVersion = versionStringToLong( select().storeVersion());
writeCorrectMetaDataRecord( file, positions, storeVersion );

List<Long> actualValues = new ArrayList<>();
try ( MetaDataStore store = newMetaDataStore() )
Expand All @@ -595,8 +595,16 @@ public void mustSupportScanningAllRecords() throws Exception
} );
}

List<Long> expectedValues = Arrays.stream( positions ).map(
p -> p.ordinal() + 1L ).collect( Collectors.toList() );
List<Long> expectedValues = Arrays.stream( positions ).map( p -> {
if ( p == MetaDataStore.Position.STORE_VERSION )
{
return storeVersion;
}
else
{
return p.ordinal() + 1L;
}
} ).collect( Collectors.toList() );

assertThat( actualValues, is( expectedValues ) );
}
Expand All @@ -608,10 +616,8 @@ public void mustSupportScanningAllRecordsWithRecordCursor() throws Exception
fs.mkdir( STORE_DIR );
fs.create( file ).close();
MetaDataStore.Position[] positions = MetaDataStore.Position.values();
for ( MetaDataStore.Position position : positions )
{
MetaDataStore.setRecord( pageCache, file, position, position.ordinal() + 1 );
}
long storeVersion = versionStringToLong( select().storeVersion());
writeCorrectMetaDataRecord( file, positions, storeVersion );

List<Long> actualValues = new ArrayList<>();
try ( MetaDataStore store = newMetaDataStore() )
Expand All @@ -631,9 +637,34 @@ public void mustSupportScanningAllRecordsWithRecordCursor() throws Exception
}
}

List<Long> expectedValues = Arrays.stream( positions ).map(
p -> p.ordinal() + 1L ).collect( Collectors.toList() );
List<Long> expectedValues = Arrays.stream( positions ).map( p -> {
if ( p == MetaDataStore.Position.STORE_VERSION )
{
return storeVersion;
}
else
{
return p.ordinal() + 1L;
}
} ).collect( Collectors.toList() );


assertThat( actualValues, is( expectedValues ) );
}

private void writeCorrectMetaDataRecord( File file, MetaDataStore.Position[] positions, long storeVersion )
throws IOException
{
for ( MetaDataStore.Position position : positions )
{
if ( position == MetaDataStore.Position.STORE_VERSION )
{
MetaDataStore.setRecord( pageCache, file, position, storeVersion );
}
else
{
MetaDataStore.setRecord( pageCache, file, position, position.ordinal() + 1 );
}
}
}
}
17 changes: 11 additions & 6 deletions community/kernel/src/test/java/org/neo4j/kernel/impl/store/NeoStoresTest.java
View file
Expand Up @@ -61,6 +61,7 @@
import org.neo4j.kernel.impl.core.RelationshipTypeToken;
import org.neo4j.kernel.impl.storageengine.impl.recordstorage.RecordStorageEngine;
import org.neo4j.kernel.impl.store.MetaDataStore.Position;
import org.neo4j.kernel.impl.store.format.InternalRecordFormatSelector;
import org.neo4j.kernel.impl.store.format.lowlimit.DynamicRecordFormat;
import org.neo4j.kernel.impl.store.id.DefaultIdGeneratorFactory;
import org.neo4j.kernel.impl.store.record.PropertyBlock;
Expand Down Expand Up @@ -92,6 +93,7 @@
import static org.junit.Assert.assertTrue;
import static org.neo4j.helpers.collection.MapUtil.stringMap;
import static org.neo4j.kernel.impl.store.RecordStore.getRecord;
import static org.neo4j.kernel.impl.store.format.InternalRecordFormatSelector.select;
import static org.neo4j.kernel.impl.store.format.lowlimit.MetaDataRecordFormat.FIELD_NOT_PRESENT;
import static org.neo4j.kernel.impl.store.record.RecordLoad.NORMAL;

Expand Down Expand Up @@ -505,15 +507,15 @@ public void shouldNotReadNonRecordDataAsRecord() throws Exception
FileSystemAbstraction fileSystem = fs.get();
File neoStoreDir = new File( "/tmp/graph.db/neostore" ).getAbsoluteFile();
StoreFactory factory = new StoreFactory( fileSystem, neoStoreDir, pageCache, NullLogProvider.getInstance() );

long recordVersion = MetaDataStore.versionStringToLong( InternalRecordFormatSelector.select().storeVersion() );
try ( NeoStores neoStores = factory.openAllNeoStores( true ) )
{
MetaDataStore metaDataStore = neoStores.getMetaDataStore();
metaDataStore.setCreationTime( 3 );
metaDataStore.setRandomNumber( 4 );
metaDataStore.setCurrentLogVersion( 5 );
metaDataStore.setLastCommittedAndClosedTransactionId( 6, 0, 0, 0 );
metaDataStore.setStoreVersion( 7 );
metaDataStore.setStoreVersion( recordVersion );
metaDataStore.setGraphNextProp( 8 );
metaDataStore.setLatestConstraintIntroducingTx( 9 );
}
Expand All @@ -523,16 +525,19 @@ public void shouldNotReadNonRecordDataAsRecord() throws Exception
{
channel.position( 0 );
channel.write( ByteBuffer.wrap( UTF8.encode( "This is some data that is not a record." ) ) );

}

MetaDataStore.setRecord( pageCache, file, Position.STORE_VERSION, recordVersion );

try ( NeoStores neoStores = factory.openAllNeoStores() )
{
MetaDataStore metaDataStore = neoStores.getMetaDataStore();
assertEquals( FIELD_NOT_PRESENT, metaDataStore.getCreationTime() );
assertEquals( FIELD_NOT_PRESENT, metaDataStore.getRandomNumber() );
assertEquals( FIELD_NOT_PRESENT, metaDataStore.getCurrentLogVersion() );
assertEquals( FIELD_NOT_PRESENT, metaDataStore.getLastCommittedTransactionId() );
assertEquals( FIELD_NOT_PRESENT, metaDataStore.getStoreVersion() );
assertEquals( recordVersion, metaDataStore.getStoreVersion() );
assertEquals( 8, metaDataStore.getGraphNextProp() );
assertEquals( 9, metaDataStore.getLatestConstraintIntroducingTx() );
}
Expand Down Expand Up @@ -614,15 +619,15 @@ public void shouldAddUpgradeFieldsToTheNeoStoreIfNotPresent() throws IOException
FileSystemAbstraction fileSystem = fs.get();
File neoStoreDir = new File( "/tmp/graph.db/neostore" ).getAbsoluteFile();
StoreFactory factory = new StoreFactory( fileSystem, neoStoreDir, pageCache, NullLogProvider.getInstance() );

long recordVersion = MetaDataStore.versionStringToLong( select().storeVersion() );
try ( NeoStores neoStores = factory.openAllNeoStores( true ) )
{
MetaDataStore metaDataStore = neoStores.getMetaDataStore();
metaDataStore.setCreationTime( 3 );
metaDataStore.setRandomNumber( 4 );
metaDataStore.setCurrentLogVersion( 5 );
metaDataStore.setLastCommittedAndClosedTransactionId( 6, 0, 0, 0 );
metaDataStore.setStoreVersion( 7 );
metaDataStore.setStoreVersion( recordVersion );
metaDataStore.setGraphNextProp( 8 );
metaDataStore.setLatestConstraintIntroducingTx( 9 );
}
Expand All @@ -643,7 +648,7 @@ public void shouldAddUpgradeFieldsToTheNeoStoreIfNotPresent() throws IOException
assertEquals( 4, metaDataStore.getRandomNumber() );
assertEquals( 5, metaDataStore.getCurrentLogVersion() );
assertEquals( 6, metaDataStore.getLastCommittedTransactionId() );
assertEquals( 7, metaDataStore.getStoreVersion() );
assertEquals( recordVersion, metaDataStore.getStoreVersion() );
assertEquals( 8, metaDataStore.getGraphNextProp() );
assertEquals( 9, metaDataStore.getLatestConstraintIntroducingTx() );
assertEquals( new TransactionId( 10, 11 ), metaDataStore.getUpgradeTransaction() );
Expand Down
8 changes: 6 additions & 2 deletions ...el/src/test/java/org/neo4j/unsafe/impl/batchimport/store/BatchingTokenRepositoryTest.java
View file
Expand Up @@ -22,18 +22,21 @@
import org.junit.Test;

import org.neo4j.kernel.impl.store.NodeLabelsField;
import org.neo4j.kernel.impl.store.TokenStore;
import org.neo4j.unsafe.impl.batchimport.store.BatchingTokenRepository.BatchingLabelTokenRepository;

import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.mock;

public class BatchingTokenRepositoryTest
{
@Test
public void shouldDedupLabelIds() throws Exception
{
// GIVEN
BatchingLabelTokenRepository repo = new BatchingLabelTokenRepository( null );
@SuppressWarnings( "unchecked" )
BatchingLabelTokenRepository repo = new BatchingLabelTokenRepository( mock( TokenStore.class ) );

// WHEN
long[] ids = repo.getOrCreateIds( new String[] {"One", "Two", "One"} );
Expand All @@ -46,7 +49,8 @@ public void shouldDedupLabelIds() throws Exception
public void shouldSortLabelIds() throws Exception
{
// GIVEN
BatchingLabelTokenRepository repo = new BatchingLabelTokenRepository( null );
@SuppressWarnings( "unchecked" )
BatchingLabelTokenRepository repo = new BatchingLabelTokenRepository( mock( TokenStore.class ) );
long[] expected = new long[] {
repo.getOrCreateId( "One" ),
repo.getOrCreateId( "Two" ),
Expand Down

0 comments on commit 0b192fd

Please sign in to comment.