diff --git a/community/dbms/src/test/java/org/neo4j/commandline/dbms/StoreInfoCommandTest.java b/community/dbms/src/test/java/org/neo4j/commandline/dbms/StoreInfoCommandTest.java index 48e091e20933..0dd5e710a51a 100644 --- a/community/dbms/src/test/java/org/neo4j/commandline/dbms/StoreInfoCommandTest.java +++ b/community/dbms/src/test/java/org/neo4j/commandline/dbms/StoreInfoCommandTest.java @@ -40,8 +40,7 @@ import org.neo4j.kernel.impl.store.MetaDataStore; import org.neo4j.kernel.impl.store.format.RecordFormatSelector; import org.neo4j.kernel.impl.store.format.RecordFormats; -import org.neo4j.kernel.impl.store.format.standard.StandardV2_2; -import org.neo4j.kernel.internal.Version; +import org.neo4j.kernel.impl.store.format.standard.StandardV2_3; import org.neo4j.test.rule.PageCacheRule; import org.neo4j.test.rule.TestDirectory; import org.neo4j.test.rule.fs.DefaultFileSystemRule; @@ -148,7 +147,7 @@ public void readsLatestStoreVersionCorrectly() throws Exception @Test public void readsOlderStoreVersionCorrectly() throws Exception { - prepareNeoStoreFile( StandardV2_2.RECORD_FORMATS.storeVersion() ); + prepareNeoStoreFile( StandardV2_3.RECORD_FORMATS.storeVersion() ); execute( databaseDirectory.toString() ); @@ -156,9 +155,9 @@ public void readsOlderStoreVersionCorrectly() throws Exception assertEquals( Arrays.asList( - "Store format version: v0.A.5", - "Store format introduced in: 2.2.0", - "Store format superseded in: 2.3.0" ), + "Store format version: v0.A.6", + "Store format introduced in: 2.3.0", + "Store format superseded in: 3.0.0" ), outCaptor.getAllValues() ); } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/labelscan/NativeLabelScanStore.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/labelscan/NativeLabelScanStore.java index 8f4643fba03b..98f5cc112b32 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/labelscan/NativeLabelScanStore.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/labelscan/NativeLabelScanStore.java @@ -26,8 +26,8 @@ import java.io.UncheckedIOException; import java.nio.file.NoSuchFileException; import java.util.function.Consumer; +import java.util.Optional; import java.util.function.IntFunction; -import java.util.stream.Collectors; import org.neo4j.cursor.RawCursor; import org.neo4j.graphdb.ResourceIterator; @@ -49,7 +49,6 @@ import org.neo4j.storageengine.api.schema.LabelScanReader; import static org.neo4j.helpers.Format.duration; -import static org.neo4j.helpers.collection.Iterables.single; import static org.neo4j.helpers.collection.Iterators.asResourceIterator; import static org.neo4j.helpers.collection.Iterators.iterator; import static org.neo4j.helpers.collection.MapUtil.map; @@ -325,8 +324,7 @@ public boolean hasStore() throws IOException { try { - storeFileHandle(); - return true; + return storeFileHandle().isPresent(); } catch ( NoSuchFileException e ) { @@ -334,9 +332,9 @@ public boolean hasStore() throws IOException } } - private FileHandle storeFileHandle() throws IOException + private Optional storeFileHandle() throws IOException { - return single( pageCache.streamFilesRecursive( storeFile ).collect( Collectors.toList() ) ); + return pageCache.streamFilesRecursive( storeFile ).findFirst(); } /** @@ -389,7 +387,11 @@ private void dropStrict() throws IOException index.close(); index = null; } - storeFileHandle().delete(); + Optional fileHandle = storeFileHandle(); + if ( fileHandle.isPresent() ) + { + fileHandle.get().delete(); + } } /** diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/DatabaseMigrator.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/DatabaseMigrator.java index cc23e90d583b..b0b9cc94e8b6 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/DatabaseMigrator.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/DatabaseMigrator.java @@ -29,7 +29,6 @@ import org.neo4j.kernel.impl.api.scan.LabelScanStoreProvider; import org.neo4j.kernel.impl.logging.LogService; import org.neo4j.kernel.impl.store.format.RecordFormats; -import org.neo4j.kernel.impl.storemigration.legacystore.LegacyStoreVersionCheck; import org.neo4j.kernel.impl.storemigration.monitoring.MigrationProgressMonitor; import org.neo4j.kernel.impl.storemigration.participant.LegacyIndexMigrator; import org.neo4j.kernel.impl.storemigration.participant.StoreMigrator; @@ -82,7 +81,7 @@ public void migrate( File storeDir ) { LogProvider logProvider = logService.getInternalLogProvider(); UpgradableDatabase upgradableDatabase = - new UpgradableDatabase( fs, new StoreVersionCheck( pageCache ), new LegacyStoreVersionCheck( fs ), + new UpgradableDatabase( fs, new StoreVersionCheck( pageCache ), format ); StoreUpgrader storeUpgrader = new StoreUpgrader( upgradableDatabase, progressMonitor, config, fs, pageCache, logProvider ); @@ -90,7 +89,7 @@ public void migrate( File storeDir ) StoreMigrationParticipant schemaMigrator = schemaIndexProvider.storeMigrationParticipant( fs, pageCache, labelScanStoreProvider ); LegacyIndexMigrator legacyIndexMigrator = new LegacyIndexMigrator( fs, indexProviders, logProvider ); - StoreMigrator storeMigrator = new StoreMigrator( fs, pageCache, config, logService, schemaIndexProvider ); + StoreMigrator storeMigrator = new StoreMigrator( fs, pageCache, config, logService ); storeUpgrader.addParticipant( schemaMigrator ); storeUpgrader.addParticipant( legacyIndexMigrator ); diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/UpgradableDatabase.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/UpgradableDatabase.java index 27d1e69af3f1..130253d748d3 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/UpgradableDatabase.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/UpgradableDatabase.java @@ -24,8 +24,6 @@ import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.kernel.impl.store.MetaDataStore; -import org.neo4j.kernel.impl.store.StoreFile; -import org.neo4j.kernel.impl.store.format.Capability; import org.neo4j.kernel.impl.store.format.FormatFamily; import org.neo4j.kernel.impl.store.format.RecordFormatSelector; import org.neo4j.kernel.impl.store.format.RecordFormats; @@ -36,7 +34,6 @@ import org.neo4j.kernel.impl.storemigration.StoreUpgrader.UpgradingStoreVersionNotFoundException; import org.neo4j.kernel.impl.storemigration.StoreVersionCheck.Result; import org.neo4j.kernel.impl.storemigration.StoreVersionCheck.Result.Outcome; -import org.neo4j.kernel.impl.storemigration.legacystore.LegacyStoreVersionCheck; import org.neo4j.kernel.impl.transaction.log.PhysicalLogFiles; import org.neo4j.kernel.impl.transaction.log.ReadableClosablePositionAwareChannel; import org.neo4j.kernel.impl.transaction.log.entry.LogEntryReader; @@ -52,15 +49,13 @@ public class UpgradableDatabase { private final FileSystemAbstraction fs; private final StoreVersionCheck storeVersionCheck; - private final LegacyStoreVersionCheck legacyStoreVersionCheck; private final RecordFormats format; public UpgradableDatabase( FileSystemAbstraction fs, - StoreVersionCheck storeVersionCheck, LegacyStoreVersionCheck legacyStoreVersionCheck, RecordFormats format ) + StoreVersionCheck storeVersionCheck, RecordFormats format ) { this.fs = fs; this.storeVersionCheck = storeVersionCheck; - this.legacyStoreVersionCheck = legacyStoreVersionCheck; this.format = format; } @@ -109,9 +104,7 @@ public RecordFormats checkUpgradeable( File storeDirectory ) } else { - result = fromFormat.hasCapability( Capability.VERSION_TRAILERS ) - ? checkCleanShutDownByVersionTrailer( storeDirectory, fromFormat ) - : checkCleanShutDownByCheckPoint( storeDirectory ); + result = checkCleanShutDownByCheckPoint( storeDirectory ); if ( result.outcome.isSuccessful() ) { return fromFormat; @@ -164,28 +157,12 @@ private Result checkCleanShutDownByCheckPoint( File storeDirectory ) return new Result( Result.Outcome.storeNotCleanlyShutDown, null, null ); } - private Result checkCleanShutDownByVersionTrailer( File storeDirectory, RecordFormats fromFormat ) - { - Result result = null; - for ( StoreFile store : StoreFile.legacyStoreFilesForVersion( fromFormat.storeVersion() ) ) - { - String expectedVersion = store.forVersion( fromFormat.storeVersion() ); - File storeFile = new File( storeDirectory, store.storeFileName() ); - result = legacyStoreVersionCheck.hasVersion( storeFile, expectedVersion, store.isOptional() ); - if ( !result.outcome.isSuccessful() ) - { - break; - } - } - return result; - } - private String getPathToStoreFile( File storeDirectory, Result result ) { return new File( storeDirectory, result.storeFilename ).getAbsolutePath(); } - public boolean hasCurrentVersion( File storeDir ) + boolean hasCurrentVersion( File storeDir ) { File neoStore = new File( storeDir, MetaDataStore.DEFAULT_NAME ); Result result = storeVersionCheck.hasVersion( neoStore, format.storeVersion() ); diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/legacystore/LegacyStoreVersionCheck.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/legacystore/LegacyStoreVersionCheck.java deleted file mode 100644 index beaef77f9132..000000000000 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/legacystore/LegacyStoreVersionCheck.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Copyright (c) 2002-2017 "Neo Technology," - * Network Engine for Objects in Lund AB [http://neotechnology.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package org.neo4j.kernel.impl.storemigration.legacystore; - -import java.io.File; -import java.io.IOException; -import java.nio.ByteBuffer; - -import org.neo4j.io.fs.FileSystemAbstraction; -import org.neo4j.io.fs.StoreChannel; -import org.neo4j.kernel.impl.storemigration.StoreVersionCheck.Result; -import org.neo4j.kernel.impl.storemigration.StoreVersionCheck.Result.Outcome; -import org.neo4j.string.UTF8; - -public class LegacyStoreVersionCheck -{ - private final FileSystemAbstraction fs; - - public LegacyStoreVersionCheck( FileSystemAbstraction fs ) - { - this.fs = fs; - } - - public Result hasVersion( File storeFile, String expectedVersion, boolean optional ) - { - StoreChannel fileChannel = null; - String storeFilename = storeFile.getName(); - byte[] expectedVersionBytes = UTF8.encode( expectedVersion ); - try - { - if ( !fs.fileExists( storeFile ) ) - { - if ( optional ) - { - return new Result( Outcome.ok, null, storeFilename ); - } - - return new Result( Outcome.missingStoreFile, null, storeFilename ); - } - - fileChannel = fs.open( storeFile, "r" ); - if ( fileChannel.size() < expectedVersionBytes.length ) - { - return new Result( Outcome.storeVersionNotFound, null, storeFilename ); - } - - String actualVersion = readVersion( fileChannel, expectedVersionBytes.length ); - if ( !expectedVersion.equals( actualVersion ) ) - { - return new Result( Outcome.unexpectedStoreVersion, actualVersion, storeFilename ); - } - } - catch ( IOException e ) - { - throw new RuntimeException( e ); - } - finally - { - if ( fileChannel != null ) - { - try - { - fileChannel.close(); - } - catch ( IOException e ) - { - // Ignore exception on close - } - } - } - return new Result( Outcome.ok, null, storeFilename ); - } - - private String readVersion( StoreChannel fileChannel, int bytesToRead ) throws IOException - { - fileChannel.position( fileChannel.size() - bytesToRead ); - byte[] foundVersionBytes = new byte[bytesToRead]; - fileChannel.read( ByteBuffer.wrap( foundVersionBytes ) ); - return UTF8.decode( foundVersionBytes ); - } -} diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigrator.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigrator.java index d6a555545653..4b8619fff410 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigrator.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigrator.java @@ -42,18 +42,15 @@ import java.util.function.Supplier; import java.util.stream.StreamSupport; -import org.neo4j.helpers.collection.Iterables; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.FileHandle; import org.neo4j.io.pagecache.PageCache; import org.neo4j.io.pagecache.PageCursor; import org.neo4j.io.pagecache.PagedFile; -import org.neo4j.kernel.api.index.SchemaIndexProvider; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.api.store.PropertyCursor; import org.neo4j.kernel.impl.locking.LockService; import org.neo4j.kernel.impl.logging.LogService; -import org.neo4j.kernel.impl.store.CountsComputer; import org.neo4j.kernel.impl.store.MetaDataStore; import org.neo4j.kernel.impl.store.MetaDataStore.Position; import org.neo4j.kernel.impl.store.NeoStores; @@ -64,16 +61,13 @@ import org.neo4j.kernel.impl.store.StoreFile; import org.neo4j.kernel.impl.store.StoreType; import org.neo4j.kernel.impl.store.TransactionId; -import org.neo4j.kernel.impl.store.counts.CountsTracker; import org.neo4j.kernel.impl.store.format.CapabilityType; import org.neo4j.kernel.impl.store.format.FormatFamily; import org.neo4j.kernel.impl.store.format.RecordFormats; import org.neo4j.kernel.impl.store.format.standard.MetaDataRecordFormat; import org.neo4j.kernel.impl.store.format.standard.NodeRecordFormat; import org.neo4j.kernel.impl.store.format.standard.RelationshipRecordFormat; -import org.neo4j.kernel.impl.store.format.standard.StandardV2_0; -import org.neo4j.kernel.impl.store.format.standard.StandardV2_1; -import org.neo4j.kernel.impl.store.format.standard.StandardV2_2; +import org.neo4j.kernel.impl.store.format.standard.StandardV2_3; import org.neo4j.kernel.impl.store.id.ReadOnlyIdGeneratorFactory; import org.neo4j.kernel.impl.store.record.NodeRecord; import org.neo4j.kernel.impl.store.record.PrimitiveRecord; @@ -84,14 +78,12 @@ import org.neo4j.kernel.impl.storemigration.StoreMigratorCheckPointer; import org.neo4j.kernel.impl.storemigration.StoreUpgrader; import org.neo4j.kernel.impl.storemigration.legacylogs.LegacyLogs; -import org.neo4j.kernel.impl.storemigration.legacystore.v21.propertydeduplication.PropertyDeduplicator; import org.neo4j.kernel.impl.storemigration.monitoring.MigrationProgressMonitor; import org.neo4j.kernel.impl.storemigration.monitoring.SilentMigrationProgressMonitor; import org.neo4j.kernel.impl.transaction.log.LogPosition; import org.neo4j.kernel.impl.transaction.log.PhysicalLogFiles; import org.neo4j.kernel.impl.transaction.log.TransactionIdStore; import org.neo4j.kernel.impl.util.CustomIOConfigValidator; -import org.neo4j.kernel.lifecycle.Lifespan; import org.neo4j.logging.NullLogProvider; import org.neo4j.storageengine.api.txstate.PropertyContainerState; import org.neo4j.unsafe.impl.batchimport.AdditionalInitialIds; @@ -110,7 +102,6 @@ import org.neo4j.unsafe.impl.batchimport.staging.ExecutionMonitor; import static java.util.Arrays.asList; -import static org.neo4j.kernel.impl.store.MetaDataStore.DEFAULT_NAME; import static org.neo4j.kernel.impl.store.format.Capability.VERSION_TRAILERS; import static org.neo4j.kernel.impl.store.format.RecordFormatSelector.selectForVersion; import static org.neo4j.kernel.impl.store.format.standard.MetaDataRecordFormat.FIELD_NOT_PRESENT; @@ -137,7 +128,7 @@ */ public class StoreMigrator extends AbstractStoreMigrationParticipant { - // Developers: There is a benchmark, storemigrate-benchmark, that generates large stores and benchmarks + // Developers: There is a benchmark, storemigrator-benchmark, that generates large stores and benchmarks // the upgrade process. Please utilize that when writing upgrade code to ensure the code is fast enough to // complete upgrades in a reasonable time period. @@ -150,23 +141,21 @@ public class StoreMigrator extends AbstractStoreMigrationParticipant private final LegacyLogs legacyLogs; private final FileSystemAbstraction fileSystem; private final PageCache pageCache; - private final SchemaIndexProvider schemaIndexProvider; public StoreMigrator( FileSystemAbstraction fileSystem, PageCache pageCache, Config config, - LogService logService, SchemaIndexProvider schemaIndexProvider ) + LogService logService ) { - this( fileSystem, pageCache, config, logService, schemaIndexProvider, new LegacyLogs( fileSystem ) ); + this( fileSystem, pageCache, config, logService, new LegacyLogs( fileSystem ) ); } public StoreMigrator( FileSystemAbstraction fileSystem, PageCache pageCache, Config config, - LogService logService, SchemaIndexProvider schemaIndexProvider, LegacyLogs legacyLogs ) + LogService logService, LegacyLogs legacyLogs ) { super( "Store files" ); this.fileSystem = fileSystem; this.pageCache = pageCache; this.config = config; this.logService = logService; - this.schemaIndexProvider = schemaIndexProvider; this.legacyLogs = legacyLogs; } @@ -174,9 +163,7 @@ public StoreMigrator( FileSystemAbstraction fileSystem, PageCache pageCache, Con public void migrate( File storeDir, File migrationDir, MigrationProgressMonitor.Section progressMonitor, String versionToMigrateFrom, String versionToMigrateTo ) throws IOException { - if ( versionToMigrateFrom.equals( StandardV2_0.STORE_VERSION ) || - versionToMigrateFrom.equals( StandardV2_1.STORE_VERSION ) || - versionToMigrateFrom.equals( StandardV2_2.STORE_VERSION ) ) + if ( versionToMigrateFrom.equals( StandardV2_3.STORE_VERSION ) ) { // These versions are not supported for block devices. CustomIOConfigValidator.assertCustomIOConfigNotUsed( config, CUSTOM_IO_EXCEPTION_MESSAGE ); @@ -224,16 +211,6 @@ public void migrate( File storeDir, File migrationDir, MigrationProgressMonitor. lastTxId, lastTxInfo.checksum(), lastTxLogPosition.getLogVersion(), lastTxLogPosition.getByteOffset(), progressMonitor, oldFormat, newFormat ); } - - if ( versionToMigrateFrom.equals( StandardV2_1.STORE_VERSION ) ) - { - removeDuplicateEntityProperties( storeDir, migrationDir, pageCache, schemaIndexProvider, oldFormat ); - } - - // DO NOT migrate logs. LegacyLogs is able to migrate logs, but only changes its format, not any - // contents of it, and since the record format has changed there would be a mismatch between the - // commands in the log and the contents in the store. If log migration is to be performed there - // must be a proper translation happening while doing so. } private boolean isDifferentCapabilities( RecordFormats oldFormat, RecordFormats newFormat ) @@ -372,57 +349,6 @@ private LogPosition extractTransactionLogPosition( File neoStore, File storeDir, } - private void removeDuplicateEntityProperties( File storeDir, File migrationDir, PageCache pageCache, - SchemaIndexProvider schemaIndexProvider, RecordFormats oldFormat ) - throws IOException - { - StoreFile.fileOperation( COPY, fileSystem, storeDir, migrationDir, Iterables.iterable( - StoreFile.PROPERTY_STORE, - StoreFile.PROPERTY_KEY_TOKEN_NAMES_STORE, - StoreFile.PROPERTY_KEY_TOKEN_STORE, - StoreFile.PROPERTY_ARRAY_STORE, - StoreFile.PROPERTY_STRING_STORE, - StoreFile.NODE_STORE, - StoreFile.NODE_LABEL_STORE, - StoreFile.SCHEMA_STORE ), false, ExistingTargetStrategy.SKIP, StoreFileType.STORE ); - - // copy ids only if present - StoreFile.fileOperation( COPY, fileSystem, storeDir, migrationDir, Iterables.iterable( - StoreFile.PROPERTY_STORE, - StoreFile.PROPERTY_KEY_TOKEN_NAMES_STORE, - StoreFile.PROPERTY_KEY_TOKEN_STORE, - StoreFile.NODE_STORE ), true, ExistingTargetStrategy.SKIP, StoreFileType.ID ); - - // let's remove trailers here on the copied files since the new code doesn't remove them since in 2.3 - // there are no store trailers - StoreFile.removeTrailers( oldFormat.storeVersion(), fileSystem, migrationDir, pageCache.pageSize() ); - - new PropertyDeduplicator( fileSystem, migrationDir, pageCache, schemaIndexProvider ) - .deduplicateProperties(); - } - - private void rebuildCountsFromScratch( File storeDir, long lastTxId, PageCache pageCache ) - { - final File storeFileBase = new File( storeDir, MetaDataStore.DEFAULT_NAME + StoreFactory.COUNTS_STORE ); - - StoreFactory storeFactory = new StoreFactory( storeDir, pageCache, fileSystem, NullLogProvider.getInstance() ); - try ( NeoStores neoStores = storeFactory.openAllNeoStores() ) - { - NodeStore nodeStore = neoStores.getNodeStore(); - RelationshipStore relationshipStore = neoStores.getRelationshipStore(); - try ( Lifespan life = new Lifespan() ) - { - int highLabelId = (int) neoStores.getLabelTokenStore().getHighId(); - int highRelationshipTypeId = (int) neoStores.getRelationshipTypeTokenStore().getHighId(); - CountsComputer initializer = new CountsComputer( - lastTxId, nodeStore, relationshipStore, highLabelId, highRelationshipTypeId ); - life.add( new CountsTracker( - logService.getInternalLogProvider(), fileSystem, pageCache, config, storeFileBase ) - .setInitializer( initializer ) ); - } - } - } - private void migrateWithBatchImporter( File storeDir, File migrationDir, long lastTxId, long lastTxChecksum, long lastTxLogVersion, long lastTxLogByteOffset, MigrationProgressMonitor.Section progressMonitor, RecordFormats oldFormat, RecordFormats newFormat ) @@ -535,7 +461,7 @@ private void prepareBatchImportMigration( File storeDir, File migrationDir, Reco try ( PagedFile fromFile = pageCache.map( fromPath, pageSize ); PagedFile toFile = pageCache.map( toPath, pageSize, StandardOpenOption.CREATE ); PageCursor fromCursor = fromFile.io( 0L, PagedFile.PF_SHARED_READ_LOCK ); - PageCursor toCursor = toFile.io( 0L, PagedFile.PF_SHARED_WRITE_LOCK ); ) + PageCursor toCursor = toFile.io( 0L, PagedFile.PF_SHARED_WRITE_LOCK ) ) { while ( fromCursor.next() ) { @@ -761,24 +687,6 @@ public void moveMigratedFiles( File migrationDir, File storeDir, String versionT } } - @Override - public void rebuildCounts( File storeDir, String versionToMigrateFrom, String versionToMigrateTo ) throws - IOException - { - if ( StandardV2_1.STORE_VERSION.equals( versionToMigrateFrom ) || - StandardV2_2.STORE_VERSION.equals( versionToMigrateFrom ) ) - { - // create counters from scratch - Iterable countsStoreFiles = - Iterables.iterable( StoreFile.COUNTS_STORE_LEFT, StoreFile.COUNTS_STORE_RIGHT ); - StoreFile.fileOperation( DELETE, fileSystem, storeDir, storeDir, - countsStoreFiles, true, null, StoreFileType.STORE ); - File neoStore = new File( storeDir, DEFAULT_NAME ); - long lastTxId = MetaDataStore.getRecord( pageCache, neoStore, Position.LAST_TRANSACTION_ID ); - rebuildCountsFromScratch( storeDir, lastTxId, pageCache ); - } - } - private void updateOrAddNeoStoreFieldsAsPartOfMigration( File migrationDir, File storeDir, String versionToMigrateTo, LogPosition lastClosedTxLogPosition ) throws IOException { diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/format/standard/RelationshipGroupRecordFormatTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/format/standard/RelationshipGroupRecordFormatTest.java index c7728caa79bf..aa9ee7b54d16 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/format/standard/RelationshipGroupRecordFormatTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/format/standard/RelationshipGroupRecordFormatTest.java @@ -32,9 +32,8 @@ import org.neo4j.kernel.impl.store.format.RecordFormats; import org.neo4j.kernel.impl.store.record.RelationshipGroupRecord; -import static org.junit.Assert.assertEquals; import static java.util.Arrays.asList; - +import static org.junit.Assert.assertEquals; import static org.neo4j.kernel.impl.store.NoStoreHeader.NO_STORE_HEADER; import static org.neo4j.kernel.impl.store.record.RecordLoad.NORMAL; @@ -44,8 +43,7 @@ public class RelationshipGroupRecordFormatTest @Parameters public static Collection formats() { - return asList( StandardV2_1.RECORD_FORMATS, StandardV2_2.RECORD_FORMATS, StandardV2_3.RECORD_FORMATS, - StandardV3_0.RECORD_FORMATS ); + return asList( StandardV2_3.RECORD_FORMATS, StandardV3_0.RECORD_FORMATS ); } private final RecordFormat format; diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/MigrationTestUtils.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/MigrationTestUtils.java index 05c1c28c1666..e03cb2eca64e 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/MigrationTestUtils.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/MigrationTestUtils.java @@ -22,27 +22,17 @@ import java.io.File; import java.io.IOException; import java.nio.ByteBuffer; -import java.util.function.Predicate; import org.neo4j.graphdb.mockfs.EphemeralFileSystemAbstraction; -import org.neo4j.helpers.collection.Iterables; import org.neo4j.io.fs.DefaultFileSystemAbstraction; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.fs.StoreChannel; import org.neo4j.kernel.impl.store.MetaDataStore; -import org.neo4j.kernel.impl.store.StoreFile; import org.neo4j.kernel.impl.store.format.RecordFormatSelector; -import org.neo4j.kernel.impl.store.format.standard.StandardV2_0; -import org.neo4j.kernel.impl.store.format.standard.StandardV2_1; -import org.neo4j.kernel.impl.store.format.standard.StandardV2_2; import org.neo4j.kernel.impl.store.format.standard.StandardV2_3; import org.neo4j.kernel.impl.store.format.standard.StandardV3_0; -import org.neo4j.kernel.impl.storemigration.StoreVersionCheck.Result.Outcome; -import org.neo4j.kernel.impl.storemigration.legacystore.LegacyStoreVersionCheck; -import org.neo4j.kernel.impl.storemigration.legacystore.v20.Legacy20Store; -import org.neo4j.kernel.impl.storemigration.legacystore.v21.Legacy21Store; -import org.neo4j.kernel.impl.storemigration.legacystore.v22.Legacy22Store; import org.neo4j.kernel.impl.storemigration.legacystore.v23.Legacy23Store; +import org.neo4j.kernel.impl.storemigration.legacystore.v30.Legacy30Store; import org.neo4j.kernel.impl.transaction.log.LogPosition; import org.neo4j.kernel.impl.transaction.log.PhysicalLogFiles; import org.neo4j.kernel.impl.transaction.log.ReadableClosablePositionAwareChannel; @@ -83,7 +73,7 @@ public static String makeLongString() return builder.toString(); } - public static void changeVersionNumber( FileSystemAbstraction fileSystem, File storeFile, String versionString ) + static void changeVersionNumber( FileSystemAbstraction fileSystem, File storeFile, String versionString ) throws IOException { byte[] versionBytes = UTF8.encode( versionString ); @@ -94,35 +84,7 @@ public static void changeVersionNumber( FileSystemAbstraction fileSystem, File s } } - /** - * Removes the version trailer from the store files. - */ - public static void truncateFile( FileSystemAbstraction fileSystem, File storeFile, - String suffixToDetermineTruncationLength ) throws IOException - { - byte[] versionBytes = UTF8.encode( suffixToDetermineTruncationLength ); - if ( !fileSystem.fileExists( storeFile ) ) - { - return; - } - try ( StoreChannel fileChannel = fileSystem.open( storeFile, "rw" ) ) - { - long fileSize = fileSystem.getFileSize( storeFile ); - fileChannel.truncate( Math.max( 0, fileSize - versionBytes.length ) ); - } - } - - public static void truncateAllFiles( FileSystemAbstraction fileSystem, File workingDirectory, - String version ) throws IOException - { - for ( StoreFile storeFile : StoreFile.legacyStoreFilesForVersion( version ) ) - { - File file = new File( workingDirectory, storeFile.storeFileName() ); - truncateFile( fileSystem, file, storeFile.forVersion( version ) ); - } - } - - public static void truncateToFixedLength( FileSystemAbstraction fileSystem, File storeFile, int newLength ) + static void truncateToFixedLength( FileSystemAbstraction fileSystem, File storeFile, int newLength ) throws IOException { try ( StoreChannel fileChannel = fileSystem.open( storeFile, "rw" ) ) @@ -155,23 +117,15 @@ public static void prepareSampleLegacyDatabase( String version, FileSystemAbstra workingFs.copyRecursively( resourceDirectory, workingDirectory ); } - public static File findFormatStoreDirectoryForVersion( String version, File targetDir ) throws IOException + static File findFormatStoreDirectoryForVersion( String version, File targetDir ) throws IOException { - if ( version.equals( StandardV2_3.STORE_VERSION ) ) + if ( StandardV2_3.STORE_VERSION.equals( version ) ) { return find23FormatStoreDirectory( targetDir ); } - else if ( version.equals( StandardV2_2.STORE_VERSION ) ) - { - return find22FormatStoreDirectory( targetDir ); - } - else if ( version.equals( StandardV2_1.STORE_VERSION ) ) + else if ( StandardV3_0.STORE_VERSION.equals( version ) ) { - return find21FormatStoreDirectory( targetDir ); - } - else if ( version.equals( StandardV2_0.STORE_VERSION ) ) - { - return find20FormatStoreDirectory( targetDir ); + return find30FormatStoreDirectory( targetDir ); } else { @@ -179,74 +133,14 @@ else if ( version.equals( StandardV2_0.STORE_VERSION ) ) } } - private static File find23FormatStoreDirectory( File targetDir ) throws IOException + private static File find30FormatStoreDirectory( File targetDir ) throws IOException { - return Unzip.unzip( Legacy23Store.class, "upgradeTest23Db.zip", targetDir ); + return Unzip.unzip( Legacy30Store.class, "upgradeTest30Db.zip", targetDir ); } - public static File find22FormatStoreDirectory( File targetDir ) throws IOException + public static File find23FormatStoreDirectory( File targetDir ) throws IOException { - return Unzip.unzip( Legacy22Store.class, "upgradeTest22Db.zip", targetDir ); - } - - public static File find21FormatStoreDirectory( File targetDir ) throws IOException - { - return Unzip.unzip( Legacy21Store.class, "upgradeTest21Db.zip", targetDir ); - } - - public static File find21FormatStoreDirectoryWithDuplicateProperties( File targetDir ) throws IOException - { - return Unzip.unzip( Legacy21Store.class, "with-duplicate-properties.zip", targetDir ); - } - - public static File find20FormatStoreDirectory( File targetDir ) throws IOException - { - return Unzip.unzip( Legacy20Store.class, "exampledb.zip", targetDir ); - } - - public static boolean allLegacyStoreFilesHaveVersion( FileSystemAbstraction fs, File dir, String version ) - { - final Iterable storeFilesWithGivenVersions = - Iterables.filter( ALL_EXCEPT_COUNTS_STORE, StoreFile.legacyStoreFilesForVersion( version ) ); - LegacyStoreVersionCheck legacyStoreVersionCheck = new LegacyStoreVersionCheck( fs ); - boolean success = true; - for ( StoreFile storeFile : storeFilesWithGivenVersions ) - { - File file = new File( dir, storeFile.storeFileName() ); - success &= - legacyStoreVersionCheck.hasVersion( file, version, storeFile.isOptional() ).outcome.isSuccessful(); - } - return success; - } - - public static boolean allStoreFilesHaveNoTrailer( FileSystemAbstraction fs, File dir ) - { - final Iterable storeFilesWithGivenVersions = - Iterables.filter( ALL_EXCEPT_COUNTS_STORE, StoreFile.legacyStoreFilesForVersion( StandardV3_0.STORE_VERSION ) ); - LegacyStoreVersionCheck legacyStoreVersionCheck = new LegacyStoreVersionCheck( fs ); - - boolean success = true; - for ( StoreFile storeFile : storeFilesWithGivenVersions ) - { - File file = new File( dir, storeFile.storeFileName() ); - StoreVersionCheck.Result result = - legacyStoreVersionCheck.hasVersion( file, StandardV3_0.STORE_VERSION, storeFile.isOptional() ); - success &= result.outcome == Outcome.unexpectedStoreVersion || - result.outcome == Outcome.storeVersionNotFound; - } - return success; - } - - public static boolean containsAnyStoreFiles( FileSystemAbstraction fileSystem, File directory ) - { - for ( StoreFile file : StoreFile.values() ) - { - if ( fileSystem.fileExists( new File( directory, file.storeFileName() ) ) ) - { - return true; - } - } - return false; + return Unzip.unzip( Legacy23Store.class, "upgradeTest23Db.zip", targetDir ); } public static boolean checkNeoStoreHasDefaultFormatVersion( StoreVersionCheck check, File workingDirectory ) @@ -294,14 +188,6 @@ public static void verifyFilesHaveSameContent( FileSystemAbstraction fileSystem, } } - public static File isolatedMigrationDirectoryOf( File dbDirectory ) - { - return new File( dbDirectory, "upgrade" ); - } - - private static final Predicate ALL_EXCEPT_COUNTS_STORE = - item -> item != StoreFile.COUNTS_STORE_LEFT && item != StoreFile.COUNTS_STORE_RIGHT; - public static void removeCheckPointFromTxLog( FileSystemAbstraction fileSystem, File workingDirectory ) throws IOException { diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/legacystore/LegacyStoreVersionCheckTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/legacystore/LegacyStoreVersionCheckTest.java deleted file mode 100644 index d76fe4f597d2..000000000000 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/legacystore/LegacyStoreVersionCheckTest.java +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Copyright (c) 2002-2017 "Neo Technology," - * Network Engine for Objects in Lund AB [http://neotechnology.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package org.neo4j.kernel.impl.storemigration.legacystore; - -import org.junit.Rule; -import org.junit.Test; - -import java.io.File; -import java.io.IOException; -import java.io.OutputStream; - -import org.neo4j.graphdb.mockfs.EphemeralFileSystemAbstraction; -import org.neo4j.test.rule.fs.EphemeralFileSystemRule; - -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -public class LegacyStoreVersionCheckTest -{ - @Rule - public final EphemeralFileSystemRule fs = new EphemeralFileSystemRule(); - - @Test - public void shouldReportMissingFileNotHavingSpecifiedVersion() - { - // given - File missingFile = new File( "/you/will/never/find/me" ); - LegacyStoreVersionCheck storeVersionCheck = new LegacyStoreVersionCheck( fs.get() ); - - // then - assertFalse( storeVersionCheck.hasVersion( missingFile, "version", false ).outcome.isSuccessful() ); - } - - @Test - public void shouldNotReportMissingOptionalFileNotHaveSpecifiedVersion() - { - // given - File missingFile = new File( "/you/will/never/find/me" ); - LegacyStoreVersionCheck storeVersionCheck = new LegacyStoreVersionCheck( fs.get() ); - - // then - assertTrue( storeVersionCheck.hasVersion( missingFile, "version", true ).outcome.isSuccessful() ); - } - - @Test - public void shouldReportShortFileDoesNotHaveSpecifiedVersion() throws IOException - { - // given - File shortFile = fileContaining( fs.get(), "a" ); - - LegacyStoreVersionCheck storeVersionCheck = new LegacyStoreVersionCheck( fs.get() ); - - // then - assertFalse( storeVersionCheck.hasVersion( shortFile, "version", false ).outcome.isSuccessful() ); - } - - @Test - public void shouldReportFileWithIncorrectVersion() throws IOException - { - // given - File shortFile = fileContaining( fs.get(), "versionWhichIsIncorrect" ); - - LegacyStoreVersionCheck storeVersionCheck = new LegacyStoreVersionCheck( fs.get() ); - - // then - assertFalse( storeVersionCheck.hasVersion( shortFile, "correctVersion 1", false ).outcome.isSuccessful() ); - } - - @Test - public void shouldReportFileWithCorrectVersion() throws IOException - { - // given - File shortFile = fileContaining( fs.get(), "correctVersion 1" ); - - LegacyStoreVersionCheck storeVersionCheck = new LegacyStoreVersionCheck( fs.get() ); - - // then - assertTrue( storeVersionCheck.hasVersion( shortFile, "correctVersion 1", false ).outcome.isSuccessful() ); - } - - private File fileContaining( EphemeralFileSystemAbstraction fs, String content ) throws IOException - { - File shortFile = new File( "shortFile" ); - try ( OutputStream outputStream = fs.openAsOutputStream( shortFile, true ) ) - { - outputStream.write( content.getBytes() ); - } - return shortFile; - } -} diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/legacystore/v30/Legacy30Store.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/legacystore/v30/Legacy30Store.java new file mode 100644 index 000000000000..387b6bc05005 --- /dev/null +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/legacystore/v30/Legacy30Store.java @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2002-2017 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.kernel.impl.storemigration.legacystore.v30; + +import org.neo4j.test.Unzip; + +/** + * Here as a place holder for using {@link Unzip} to find old db tarballs. + */ +public interface Legacy30Store +{ +} diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigratorIT.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigratorIT.java index 0388d5636111..6154c2af2904 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigratorIT.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigratorIT.java @@ -42,15 +42,11 @@ import org.neo4j.kernel.impl.store.TransactionId; import org.neo4j.kernel.impl.store.format.RecordFormats; import org.neo4j.kernel.impl.store.format.standard.Standard; -import org.neo4j.kernel.impl.store.format.standard.StandardV2_0; -import org.neo4j.kernel.impl.store.format.standard.StandardV2_1; -import org.neo4j.kernel.impl.store.format.standard.StandardV2_2; import org.neo4j.kernel.impl.store.format.standard.StandardV2_3; import org.neo4j.kernel.impl.storemigration.MigrationTestUtils; import org.neo4j.kernel.impl.storemigration.StoreUpgrader; import org.neo4j.kernel.impl.storemigration.StoreVersionCheck; import org.neo4j.kernel.impl.storemigration.UpgradableDatabase; -import org.neo4j.kernel.impl.storemigration.legacystore.LegacyStoreVersionCheck; import org.neo4j.kernel.impl.storemigration.monitoring.SilentMigrationProgressMonitor; import org.neo4j.kernel.impl.transaction.log.LogPosition; import org.neo4j.test.rule.PageCacheRule; @@ -60,8 +56,6 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.neo4j.helpers.collection.MapUtil.stringMap; -import static org.neo4j.kernel.impl.transaction.log.TransactionIdStore.BASE_TX_LOG_BYTE_OFFSET; -import static org.neo4j.kernel.impl.transaction.log.TransactionIdStore.BASE_TX_LOG_VERSION; import static org.neo4j.kernel.impl.transaction.log.TransactionIdStore.UNKNOWN_TX_COMMIT_TIMESTAMP; @RunWith( Parameterized.class ) @@ -91,22 +85,7 @@ public class StoreMigratorIT @Parameterized.Parameters( name = "{0}" ) public static Collection versions() { - return Arrays.asList( - new Object[]{ - StandardV2_0.STORE_VERSION, - new LogPosition( BASE_TX_LOG_VERSION, BASE_TX_LOG_BYTE_OFFSET ), - txInfoAcceptanceOnIdAndTimestamp( 1039, UNKNOWN_TX_COMMIT_TIMESTAMP ) - }, - new Object[]{ - StandardV2_1.STORE_VERSION, - new LogPosition( BASE_TX_LOG_VERSION, BASE_TX_LOG_BYTE_OFFSET ), - txInfoAcceptanceOnIdAndTimestamp( 38, UNKNOWN_TX_COMMIT_TIMESTAMP) - }, - new Object[]{ - StandardV2_2.STORE_VERSION, - new LogPosition( 2, BASE_TX_LOG_BYTE_OFFSET ), - txInfoAcceptanceOnIdAndTimestamp( 38, UNKNOWN_TX_COMMIT_TIMESTAMP ) - }, + return Arrays.asList( new Object[]{ StandardV2_3.STORE_VERSION, new LogPosition( 3, 169 ), txInfoAcceptanceOnIdAndTimestamp( 39, UNKNOWN_TX_COMMIT_TIMESTAMP ) @@ -131,19 +110,19 @@ public void shouldBeAbleToResumeMigrationOnMoving() throws Exception LogService logService = NullLogService.getInstance(); PageCache pageCache = pageCacheRule.getPageCache( fs ); UpgradableDatabase upgradableDatabase = - new UpgradableDatabase( fs, new StoreVersionCheck( pageCache ), new LegacyStoreVersionCheck( fs ), + new UpgradableDatabase( fs, new StoreVersionCheck( pageCache ), selectFormat() ); String versionToMigrateFrom = upgradableDatabase.checkUpgradeable( storeDirectory ).storeVersion(); SilentMigrationProgressMonitor progressMonitor = new SilentMigrationProgressMonitor(); - StoreMigrator migrator = new StoreMigrator( fs, pageCache, CONFIG, logService, schemaIndexProvider ); + StoreMigrator migrator = new StoreMigrator( fs, pageCache, CONFIG, logService ); File migrationDir = new File( storeDirectory, StoreUpgrader.MIGRATION_DIRECTORY ); fs.mkdirs( migrationDir ); migrator.migrate( storeDirectory, migrationDir, progressMonitor.startSection( "section" ), versionToMigrateFrom, upgradableDatabase.currentVersion() ); // WHEN simulating resuming the migration - migrator = new StoreMigrator( fs, pageCache, CONFIG, logService, schemaIndexProvider ); + migrator = new StoreMigrator( fs, pageCache, CONFIG, logService ); migrator.moveMigratedFiles( migrationDir, storeDirectory, versionToMigrateFrom, upgradableDatabase.currentVersion() ); // THEN starting the new store should be successful @@ -163,12 +142,11 @@ public void shouldBeAbleToResumeMigrationOnRebuildingCounts() throws Exception LogService logService = NullLogService.getInstance(); PageCache pageCache = pageCacheRule.getPageCache( fs ); UpgradableDatabase upgradableDatabase = - new UpgradableDatabase( fs, new StoreVersionCheck( pageCache ), new LegacyStoreVersionCheck( fs ), - selectFormat() ); + new UpgradableDatabase( fs, new StoreVersionCheck( pageCache ), selectFormat() ); String versionToMigrateFrom = upgradableDatabase.checkUpgradeable( storeDirectory ).storeVersion(); SilentMigrationProgressMonitor progressMonitor = new SilentMigrationProgressMonitor(); - StoreMigrator migrator = new StoreMigrator( fs, pageCache, CONFIG, logService, schemaIndexProvider ); + StoreMigrator migrator = new StoreMigrator( fs, pageCache, CONFIG, logService ); File migrationDir = new File( storeDirectory, StoreUpgrader.MIGRATION_DIRECTORY ); fs.mkdirs( migrationDir ); migrator.migrate( storeDirectory, migrationDir, progressMonitor.startSection( "section" ), @@ -178,7 +156,7 @@ public void shouldBeAbleToResumeMigrationOnRebuildingCounts() throws Exception // WHEN simulating resuming the migration progressMonitor = new SilentMigrationProgressMonitor(); - migrator = new StoreMigrator( fs, pageCache, CONFIG, logService, schemaIndexProvider ); + migrator = new StoreMigrator( fs, pageCache, CONFIG, logService ); migrator.rebuildCounts( storeDirectory, versionToMigrateFrom, upgradableDatabase.currentVersion() ); // THEN starting the new store should be successful @@ -198,12 +176,11 @@ public void shouldComputeTheLastTxLogPositionCorrectly() throws Throwable LogService logService = NullLogService.getInstance(); PageCache pageCache = pageCacheRule.getPageCache( fs ); UpgradableDatabase upgradableDatabase = - new UpgradableDatabase( fs, new StoreVersionCheck( pageCache ), new LegacyStoreVersionCheck( fs ), - selectFormat() ); + new UpgradableDatabase( fs, new StoreVersionCheck( pageCache ), selectFormat() ); String versionToMigrateFrom = upgradableDatabase.checkUpgradeable( storeDirectory ).storeVersion(); SilentMigrationProgressMonitor progressMonitor = new SilentMigrationProgressMonitor(); - StoreMigrator migrator = new StoreMigrator( fs, pageCache, CONFIG, logService, schemaIndexProvider ); + StoreMigrator migrator = new StoreMigrator( fs, pageCache, CONFIG, logService ); File migrationDir = new File( storeDirectory, StoreUpgrader.MIGRATION_DIRECTORY ); fs.mkdirs( migrationDir ); @@ -226,12 +203,11 @@ public void shouldComputeTheLastTxInfoCorrectly() throws Exception LogService logService = NullLogService.getInstance(); PageCache pageCache = pageCacheRule.getPageCache( fs ); UpgradableDatabase upgradableDatabase = - new UpgradableDatabase( fs, new StoreVersionCheck( pageCache ), new LegacyStoreVersionCheck( fs ), - selectFormat() ); + new UpgradableDatabase( fs, new StoreVersionCheck( pageCache ), selectFormat() ); String versionToMigrateFrom = upgradableDatabase.checkUpgradeable( storeDirectory ).storeVersion(); SilentMigrationProgressMonitor progressMonitor = new SilentMigrationProgressMonitor(); - StoreMigrator migrator = new StoreMigrator( fs, pageCache, CONFIG, logService, schemaIndexProvider ); + StoreMigrator migrator = new StoreMigrator( fs, pageCache, CONFIG, logService ); File migrationDir = new File( storeDirectory, StoreUpgrader.MIGRATION_DIRECTORY ); fs.mkdir( migrationDir ); diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigratorTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigratorTest.java index fece38bff93e..ce56deed634a 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigratorTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigratorTest.java @@ -35,18 +35,13 @@ import org.neo4j.kernel.impl.logging.LogService; import org.neo4j.kernel.impl.logging.NullLogService; import org.neo4j.kernel.impl.logging.SimpleLogService; -import org.neo4j.kernel.impl.store.MetaDataStore; import org.neo4j.kernel.impl.store.TransactionId; -import org.neo4j.kernel.impl.store.format.standard.StandardV2_2; import org.neo4j.kernel.impl.store.format.standard.StandardV3_0; import org.neo4j.kernel.impl.store.format.standard.StandardV3_2; import org.neo4j.kernel.impl.storemigration.legacylogs.LegacyLogs; import org.neo4j.kernel.impl.storemigration.monitoring.MigrationProgressMonitor; import org.neo4j.kernel.impl.transaction.log.LogPosition; import org.neo4j.kernel.impl.transaction.log.TransactionIdStore; -import org.neo4j.kernel.impl.transaction.log.PhysicalLogFiles; -import org.neo4j.kernel.impl.transaction.log.entry.VersionAwareLogEntryReader; -import org.neo4j.kernel.recovery.LatestCheckPointFinder; import org.neo4j.logging.NullLogProvider; import org.neo4j.test.rule.PageCacheRule; import org.neo4j.test.rule.RandomRule; @@ -57,7 +52,6 @@ import static org.junit.Assert.assertFalse; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import static org.neo4j.kernel.api.index.SchemaIndexProvider.NO_INDEX_PROVIDER; import static org.neo4j.kernel.impl.store.MetaDataStore.DEFAULT_NAME; import static org.neo4j.kernel.impl.store.MetaDataStore.Position.LAST_TRANSACTION_CHECKSUM; import static org.neo4j.kernel.impl.store.MetaDataStore.Position.LAST_TRANSACTION_COMMIT_TIMESTAMP; @@ -113,7 +107,7 @@ public void shouldExtractTransactionInformationFromMetaDataStore() throws Except setRecord( pageCache, neoStore, LAST_TRANSACTION_COMMIT_TIMESTAMP, timestamp ); // ... and with migrator - StoreMigrator migrator = new StoreMigrator( fileSystemRule.get(), pageCache, config, logService, NO_INDEX_PROVIDER ); + StoreMigrator migrator = new StoreMigrator( fileSystemRule.get(), pageCache, config, logService ); TransactionId actual = migrator.extractTransactionIdInformation( neoStore, storeDir, txId ); // then @@ -144,7 +138,7 @@ public void shouldExtractTransactionInformationFromLegacyLogsWhenCantFindInStore // when // ... neoStore is empty and with migrator StoreMigrator migrator = - new StoreMigrator( fileSystemRule.get(), pageCache, config, logService, schemaIndexProvider, legacyLogs ); + new StoreMigrator( fileSystemRule.get(), pageCache, config, logService, legacyLogs ); TransactionId actual = migrator.extractTransactionIdInformation( neoStore, storeDir, txId ); // then @@ -171,8 +165,7 @@ public void shouldGenerateTransactionInformationWhenLogsNotPresent() throws Exce // ... and transaction not in log when( legacyLogs.getTransactionInformation( storeDir, txId ) ).thenReturn( Optional.empty() ); // ... and with migrator - StoreMigrator migrator = new StoreMigrator( fileSystemRule.get(), pageCache, config, logService, - schemaIndexProvider ); + StoreMigrator migrator = new StoreMigrator( fileSystemRule.get(), pageCache, config, logService ); TransactionId actual = migrator.extractTransactionIdInformation( neoStore, storeDir, txId ); // then @@ -201,8 +194,7 @@ public void shouldGenerateTransactionInformationWhenLogsAreEmpty() throws Except // ... and transaction not in log when( legacyLogs.getTransactionInformation( storeDir, txId ) ).thenReturn( Optional.empty() ); // ... and with migrator - StoreMigrator migrator = new StoreMigrator( fileSystemRule.get(), pageCache, config, logService, - schemaIndexProvider ); + StoreMigrator migrator = new StoreMigrator( fileSystemRule.get(), pageCache, config, logService ); TransactionId actual = migrator.extractTransactionIdInformation( neoStore, storeDir, txId ); // then @@ -211,33 +203,6 @@ public void shouldGenerateTransactionInformationWhenLogsAreEmpty() throws Except assertEquals( TransactionIdStore.BASE_TX_COMMIT_TIMESTAMP, actual.commitTimestamp() ); } - @Test - public void checkpointBasedOnLastClosedTransactionPosition() throws IOException - { - StoreMigrator storeMigrator = newStoreMigrator(); - LogPosition lastTxLogPosition = new LogPosition( 7, 8 ); - TransactionId txInfo = new TransactionId( 100, 101, 102 ); - - File storeDir = directory.graphDbDir(); - File migrationDirectory = directory.directory( "migratedDirectory" ); - - File neoStore = new File( storeDir, DEFAULT_NAME ); - fileSystemRule.create( neoStore ).close(); - MetaDataStore.setRecord( pageCache, neoStore, MetaDataStore.Position.LOG_VERSION, 73 ); - - storeMigrator.writeLastTxLogPosition( migrationDirectory, lastTxLogPosition ); - storeMigrator.writeLastTxInformation( migrationDirectory, txInfo ); - storeMigrator.moveMigratedFiles( migrationDirectory, storeDir, - StandardV2_2.STORE_VERSION, StandardV3_0.STORE_VERSION); - - PhysicalLogFiles logFiles = new PhysicalLogFiles( storeDir, fileSystemRule ); - LatestCheckPointFinder checkPointFinder = new LatestCheckPointFinder( logFiles, fileSystemRule, - new VersionAwareLogEntryReader<>() ); - LatestCheckPointFinder.LatestCheckPoint latestCheckPoint = checkPointFinder.find( logFiles - .getHighestLogVersion() ); - assertEquals( lastTxLogPosition, latestCheckPoint.checkPoint.getLogPosition() ); - } - @Test public void writeAndReadLastTxInformation() throws IOException { @@ -286,7 +251,7 @@ public void shouldNotMigrateFilesForVersionsWithSameCapability() throws Exceptio private StoreMigrator newStoreMigrator() { return new StoreMigrator( fileSystemRule, pageCache, - Config.empty(), NullLogService.getInstance(), schemaIndexProvider ); + Config.empty(), NullLogService.getInstance() ); } private static class MySection implements MigrationProgressMonitor.Section diff --git a/community/neo4j/src/test/java/upgrade/FailToStartStoreMigratorIT.java b/community/neo4j/src/test/java/upgrade/FailToStartStoreMigratorIT.java deleted file mode 100644 index ee84c5b027ac..000000000000 --- a/community/neo4j/src/test/java/upgrade/FailToStartStoreMigratorIT.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright (c) 2002-2017 "Neo Technology," - * Network Engine for Objects in Lund AB [http://neotechnology.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package upgrade; - -import org.junit.Rule; -import org.junit.Test; - -import java.io.File; - -import org.neo4j.graphdb.factory.GraphDatabaseSettings; -import org.neo4j.kernel.impl.core.NonUniqueTokenException; -import org.neo4j.kernel.impl.storemigration.legacystore.LegacyStore; -import org.neo4j.kernel.lifecycle.LifecycleException; -import org.neo4j.test.TestGraphDatabaseFactory; -import org.neo4j.test.Unzip; -import org.neo4j.test.rule.CleanupRule; -import org.neo4j.test.rule.TestDirectory; - -import static org.hamcrest.CoreMatchers.instanceOf; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertThat; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -public class FailToStartStoreMigratorIT -{ - @Rule - public final TestDirectory testDirectory = TestDirectory.testDirectory(); - - @Rule - public final CleanupRule cleanup = new CleanupRule(); - - @Test - public void shouldFailToStartWithImproperlyUpgradedPropertyKeyStore() throws Exception - { - // given - File storeDir = testDirectory.graphDbDir(); - // a store with duplicate property keys, that was upgraded from 1.9 to 2.1.3, where no de-duplication was made - Unzip.unzip( LegacyStore.class, "v21/upgradeMissedPropKeyDup.zip", storeDir ); - - // when - try - { - new TestGraphDatabaseFactory().newEmbeddedDatabaseBuilder( storeDir ). - setConfig( GraphDatabaseSettings.allow_store_upgrade, "true" ). - newGraphDatabase(); - fail( "should have failed to start" ); - } - // then - catch ( RuntimeException e ) - { - assertThat( e.getCause(), instanceOf( LifecycleException.class ) ); - Throwable root = e.getCause().getCause(); - assertThat( root, instanceOf( NonUniqueTokenException.class ) ); - assertNull( root.getCause() ); - assertTrue( root.getMessage().startsWith( "The PropertyKey \"name\" is not unique" ) ); - } - } - -} diff --git a/community/neo4j/src/test/java/upgrade/PlatformConstraintStoreUpgradeTest.java b/community/neo4j/src/test/java/upgrade/PlatformConstraintStoreUpgradeTest.java index 148cdb95e010..09d81f7828c9 100644 --- a/community/neo4j/src/test/java/upgrade/PlatformConstraintStoreUpgradeTest.java +++ b/community/neo4j/src/test/java/upgrade/PlatformConstraintStoreUpgradeTest.java @@ -22,15 +22,17 @@ import org.junit.Before; import org.junit.Rule; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; import java.io.File; import java.io.IOException; +import java.util.Collection; +import java.util.Collections; import org.neo4j.graphdb.GraphDatabaseService; import org.neo4j.graphdb.factory.GraphDatabaseSettings; -import org.neo4j.kernel.impl.store.format.standard.StandardV2_0; -import org.neo4j.kernel.impl.store.format.standard.StandardV2_1; -import org.neo4j.kernel.impl.store.format.standard.StandardV2_2; +import org.neo4j.kernel.impl.store.format.standard.StandardV2_3; import org.neo4j.kernel.impl.storemigration.participant.StoreMigrator; import org.neo4j.test.TestGraphDatabaseFactory; import org.neo4j.test.rule.TestDirectory; @@ -41,6 +43,7 @@ import static org.neo4j.kernel.impl.pagecache.PageSwapperFactoryForTesting.TEST_PAGESWAPPER_NAME; import static org.neo4j.kernel.impl.storemigration.MigrationTestUtils.prepareSampleLegacyDatabase; +@RunWith( Parameterized.class ) public class PlatformConstraintStoreUpgradeTest { @Rule @@ -51,36 +54,34 @@ public class PlatformConstraintStoreUpgradeTest private File prepareDir; private File workingDir; - @Before - public void setup() - { - - prepareDir = storeDir.directory( "prepare" ); - workingDir = storeDir.directory( "working" ); - } + @Parameterized.Parameter( 0 ) + public String version; - @Test - public void shouldFailToStartWithCustomIOConfigurationTest20() throws IOException + @Parameterized.Parameters( name = "{0}" ) + public static Collection versions() { - String storeVersion = StandardV2_0.STORE_VERSION; - checkForStoreVersion( storeVersion ); + return Collections.singletonList( StandardV2_3.STORE_VERSION ); } - @Test - public void shouldFailToStartWithCustomIOConfigurationTest21() throws IOException +// public PlatformConstraintStoreUpgradeTest( String version ) +// { +// this.version = version; +// } + + @Before + public void setup() { - String storeVersion = StandardV2_1.STORE_VERSION; - checkForStoreVersion( storeVersion ); + prepareDir = storeDir.directory( "prepare" ); + workingDir = storeDir.directory( "working" ); } @Test - public void shouldFailToStartWithCustomIOConfigurationTest22() throws IOException + public void shouldFailToStartWithCustomIOConfigurationTest() throws IOException { - String storeVersion = StandardV2_2.STORE_VERSION; - checkForStoreVersion( storeVersion ); + checkForStoreVersion( version ); } - protected void checkForStoreVersion( String storeVersion ) throws IOException + private void checkForStoreVersion( String storeVersion ) throws IOException { prepareSampleLegacyDatabase( storeVersion, fileSystemRule.get(), workingDir, prepareDir ); try diff --git a/community/neo4j/src/test/java/upgrade/StoreUpgradeOnStartupTest.java b/community/neo4j/src/test/java/upgrade/StoreUpgradeOnStartupTest.java index 88eb952f6109..a4b7fd88975a 100644 --- a/community/neo4j/src/test/java/upgrade/StoreUpgradeOnStartupTest.java +++ b/community/neo4j/src/test/java/upgrade/StoreUpgradeOnStartupTest.java @@ -29,8 +29,8 @@ import java.io.File; import java.io.IOException; -import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import org.neo4j.consistency.checking.full.ConsistencyCheckIncompleteException; import org.neo4j.graphdb.GraphDatabaseService; @@ -38,9 +38,6 @@ import org.neo4j.helpers.Exceptions; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; -import org.neo4j.kernel.impl.store.format.standard.StandardV2_0; -import org.neo4j.kernel.impl.store.format.standard.StandardV2_1; -import org.neo4j.kernel.impl.store.format.standard.StandardV2_2; import org.neo4j.kernel.impl.store.format.standard.StandardV2_3; import org.neo4j.kernel.impl.storemigration.StoreUpgrader; import org.neo4j.kernel.impl.storemigration.StoreVersionCheck; @@ -49,17 +46,13 @@ import org.neo4j.test.rule.TestDirectory; import org.neo4j.test.rule.fs.DefaultFileSystemRule; -import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import static org.neo4j.consistency.store.StoreAssertions.assertConsistentStore; -import static org.neo4j.kernel.impl.storemigration.MigrationTestUtils.allLegacyStoreFilesHaveVersion; -import static org.neo4j.kernel.impl.storemigration.MigrationTestUtils.allStoreFilesHaveNoTrailer; import static org.neo4j.kernel.impl.storemigration.MigrationTestUtils.checkNeoStoreHasDefaultFormatVersion; import static org.neo4j.kernel.impl.storemigration.MigrationTestUtils.prepareSampleLegacyDatabase; import static org.neo4j.kernel.impl.storemigration.MigrationTestUtils.removeCheckPointFromTxLog; -import static org.neo4j.kernel.impl.storemigration.MigrationTestUtils.truncateFile; @RunWith( Parameterized.class ) public class StoreUpgradeOnStartupTest @@ -82,10 +75,7 @@ public class StoreUpgradeOnStartupTest @Parameterized.Parameters( name = "{0}" ) public static Collection versions() { - return Arrays.asList( - StandardV2_0.STORE_VERSION, - StandardV2_1.STORE_VERSION, - StandardV2_2.STORE_VERSION, + return Collections.singletonList( StandardV2_3.STORE_VERSION ); } @@ -99,8 +89,6 @@ public void setup() throws IOException check = new StoreVersionCheck( pageCache ); File prepareDirectory = testDir.directory( "prepare_" + version ); prepareSampleLegacyDatabase( version, fileSystem, workingDirectory, prepareDirectory ); - assertEquals( !StandardV2_3.STORE_VERSION.equals( version ), - allLegacyStoreFilesHaveVersion( fileSystem, workingDirectory, version ) ); } @Test @@ -113,7 +101,6 @@ public void shouldUpgradeAutomaticallyOnDatabaseStartup() throws IOException, Co // then assertTrue( "Some store files did not have the correct version", checkNeoStoreHasDefaultFormatVersion( check, workingDirectory ) ); - assertTrue( allStoreFilesHaveNoTrailer( fileSystem, workingDirectory ) ); assertConsistentStore( workingDirectory ); } @@ -121,7 +108,7 @@ public void shouldUpgradeAutomaticallyOnDatabaseStartup() throws IOException, Co public void shouldAbortOnNonCleanlyShutdown() throws Throwable { // given - makeDbNotCleanlyShutdown(); + removeCheckPointFromTxLog( fileSystem, workingDirectory ); try { // when @@ -137,19 +124,6 @@ public void shouldAbortOnNonCleanlyShutdown() throws Throwable } } - private void makeDbNotCleanlyShutdown() throws IOException - { - if ( StandardV2_3.STORE_VERSION.equals( version ) ) - { - removeCheckPointFromTxLog( fileSystem, workingDirectory ); - } - else - { - File file = new File( workingDirectory, "neostore.propertystore.db.index.keys" ); - truncateFile( fileSystem, file, "StringPropertyStore " + version ); - } - } - private GraphDatabaseService createGraphDatabaseService() { return new TestGraphDatabaseFactory() diff --git a/community/neo4j/src/test/java/upgrade/StoreUpgraderInterruptionTestIT.java b/community/neo4j/src/test/java/upgrade/StoreUpgraderInterruptionTestIT.java index 363e0d75400b..79244925fa7b 100644 --- a/community/neo4j/src/test/java/upgrade/StoreUpgraderInterruptionTestIT.java +++ b/community/neo4j/src/test/java/upgrade/StoreUpgraderInterruptionTestIT.java @@ -29,8 +29,8 @@ import java.io.File; import java.io.IOException; -import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import org.neo4j.consistency.checking.full.ConsistencyCheckIncompleteException; import org.neo4j.graphdb.GraphDatabaseService; @@ -44,15 +44,11 @@ import org.neo4j.kernel.impl.logging.LogService; import org.neo4j.kernel.impl.logging.NullLogService; import org.neo4j.kernel.impl.store.format.standard.Standard; -import org.neo4j.kernel.impl.store.format.standard.StandardV2_0; -import org.neo4j.kernel.impl.store.format.standard.StandardV2_1; -import org.neo4j.kernel.impl.store.format.standard.StandardV2_2; import org.neo4j.kernel.impl.store.format.standard.StandardV2_3; import org.neo4j.kernel.impl.storemigration.MigrationTestUtils; import org.neo4j.kernel.impl.storemigration.StoreUpgrader; import org.neo4j.kernel.impl.storemigration.StoreVersionCheck; import org.neo4j.kernel.impl.storemigration.UpgradableDatabase; -import org.neo4j.kernel.impl.storemigration.legacystore.LegacyStoreVersionCheck; import org.neo4j.kernel.impl.storemigration.monitoring.MigrationProgressMonitor; import org.neo4j.kernel.impl.storemigration.monitoring.SilentMigrationProgressMonitor; import org.neo4j.kernel.impl.storemigration.participant.SchemaIndexMigrator; @@ -70,14 +66,11 @@ import static org.junit.Assert.fail; import static org.neo4j.consistency.store.StoreAssertions.assertConsistentStore; import static org.neo4j.helpers.collection.MapUtil.stringMap; -import static org.neo4j.kernel.impl.storemigration.MigrationTestUtils.allLegacyStoreFilesHaveVersion; -import static org.neo4j.kernel.impl.storemigration.MigrationTestUtils.allStoreFilesHaveNoTrailer; import static org.neo4j.kernel.impl.storemigration.MigrationTestUtils.checkNeoStoreHasDefaultFormatVersion; @RunWith( Parameterized.class ) public class StoreUpgraderInterruptionTestIT { - private final TestDirectory directory = TestDirectory.testDirectory(); private final DefaultFileSystemRule fileSystemRule = new DefaultFileSystemRule(); private final PageCacheRule pageCacheRule = new PageCacheRule(); @@ -90,16 +83,13 @@ public class StoreUpgraderInterruptionTestIT public String version; private final SchemaIndexProvider schemaIndexProvider = new InMemoryIndexProvider(); private LabelScanStoreProvider labelScanStoreProvider; - protected static final Config CONFIG = Config.defaults().augment( + private static final Config CONFIG = Config.defaults().augment( stringMap( GraphDatabaseSettings.pagecache_memory.name(), "8m" ) ); @Parameters( name = "{0}" ) public static Collection versions() { - return Arrays.asList( - StandardV2_0.STORE_VERSION, - StandardV2_1.STORE_VERSION, - StandardV2_2.STORE_VERSION, + return Collections.singletonList( StandardV2_3.STORE_VERSION ); } @@ -125,10 +115,10 @@ public void shouldSucceedWithUpgradeAfterPreviousAttemptDiedDuringMigration() PageCache pageCache = pageCacheRule.getPageCache( fs ); StoreVersionCheck check = new StoreVersionCheck( pageCache ); UpgradableDatabase upgradableDatabase = - new UpgradableDatabase( fs, check, new LegacyStoreVersionCheck( fs ), Standard.LATEST_RECORD_FORMATS ); + new UpgradableDatabase( fs, check, Standard.LATEST_RECORD_FORMATS ); SilentMigrationProgressMonitor progressMonitor = new SilentMigrationProgressMonitor(); LogService logService = NullLogService.getInstance(); - StoreMigrator failingStoreMigrator = new StoreMigrator( fs, pageCache, CONFIG, logService, schemaIndexProvider ) + StoreMigrator failingStoreMigrator = new StoreMigrator( fs, pageCache, CONFIG, logService ) { @Override public void migrate( File sourceStoreDir, File targetStoreDir, @@ -141,9 +131,6 @@ public void migrate( File sourceStoreDir, File targetStoreDir, } }; - assertEquals( !StandardV2_3.STORE_VERSION.equals( version ), - allLegacyStoreFilesHaveVersion( fs, workingDirectory, version ) ); - try { newUpgrader( upgradableDatabase, pageCache, progressMonitor, createIndexMigrator(), failingStoreMigrator ) @@ -155,17 +142,13 @@ public void migrate( File sourceStoreDir, File targetStoreDir, assertEquals( "This upgrade is failing", e.getMessage() ); } - assertEquals( !StandardV2_3.STORE_VERSION.equals( version ), - allLegacyStoreFilesHaveVersion( fs, workingDirectory, version ) ); - progressMonitor = new SilentMigrationProgressMonitor(); - StoreMigrator migrator = new StoreMigrator( fs, pageCache, CONFIG, logService, schemaIndexProvider ); + StoreMigrator migrator = new StoreMigrator( fs, pageCache, CONFIG, logService ); SchemaIndexMigrator indexMigrator = createIndexMigrator(); newUpgrader(upgradableDatabase, pageCache, progressMonitor, indexMigrator, migrator ).migrateIfNeeded( workingDirectory ); assertTrue( checkNeoStoreHasDefaultFormatVersion( check, workingDirectory ) ); - assertTrue( allStoreFilesHaveNoTrailer( fs, workingDirectory ) ); // Since consistency checker is in read only mode we need to start/stop db to generate label scan store. startStopDatabase( workingDirectory ); @@ -187,10 +170,10 @@ public void shouldSucceedWithUpgradeAfterPreviousAttemptDiedDuringMovingFiles() PageCache pageCache = pageCacheRule.getPageCache( fs ); StoreVersionCheck check = new StoreVersionCheck( pageCache ); UpgradableDatabase upgradableDatabase = - new UpgradableDatabase( fs, check, new LegacyStoreVersionCheck( fs ), Standard.LATEST_RECORD_FORMATS ); + new UpgradableDatabase( fs, check, Standard.LATEST_RECORD_FORMATS ); SilentMigrationProgressMonitor progressMonitor = new SilentMigrationProgressMonitor(); LogService logService = NullLogService.getInstance(); - StoreMigrator failingStoreMigrator = new StoreMigrator( fs, pageCache, CONFIG, logService, schemaIndexProvider ) + StoreMigrator failingStoreMigrator = new StoreMigrator( fs, pageCache, CONFIG, logService ) { @Override public void moveMigratedFiles( File migrationDir, File storeDir, String versionToUpgradeFrom, @@ -201,9 +184,6 @@ public void moveMigratedFiles( File migrationDir, File storeDir, String versionT } }; - assertEquals( !StandardV2_3.STORE_VERSION.equals( version ), - allLegacyStoreFilesHaveVersion( fs, workingDirectory, version ) ); - try { newUpgrader( upgradableDatabase, pageCache, progressMonitor, createIndexMigrator(), failingStoreMigrator ) @@ -216,15 +196,13 @@ public void moveMigratedFiles( File migrationDir, File storeDir, String versionT } assertTrue( checkNeoStoreHasDefaultFormatVersion( check, workingDirectory ) ); - assertTrue( allStoreFilesHaveNoTrailer( fs, workingDirectory ) ); progressMonitor = new SilentMigrationProgressMonitor(); - StoreMigrator migrator = new StoreMigrator( fs, pageCache, CONFIG, logService, schemaIndexProvider ); + StoreMigrator migrator = new StoreMigrator( fs, pageCache, CONFIG, logService ); newUpgrader( upgradableDatabase, pageCache, progressMonitor, createIndexMigrator(), migrator ) .migrateIfNeeded( workingDirectory ); assertTrue( checkNeoStoreHasDefaultFormatVersion( check, workingDirectory ) ); - assertTrue( allStoreFilesHaveNoTrailer( fs, workingDirectory ) ); pageCache.close(); diff --git a/community/neo4j/src/test/java/upgrade/StoreUpgraderTest.java b/community/neo4j/src/test/java/upgrade/StoreUpgraderTest.java index 93ad92f0a876..3eef7fe1242c 100644 --- a/community/neo4j/src/test/java/upgrade/StoreUpgraderTest.java +++ b/community/neo4j/src/test/java/upgrade/StoreUpgraderTest.java @@ -38,8 +38,6 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -import org.neo4j.consistency.checking.full.ConsistencyCheckIncompleteException; -import org.neo4j.graphdb.GraphDatabaseService; import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.graphdb.mockfs.DelegatingFileSystemAbstraction; import org.neo4j.helpers.collection.MapUtil; @@ -57,9 +55,6 @@ import org.neo4j.kernel.impl.store.format.RecordFormats; import org.neo4j.kernel.impl.store.format.standard.MetaDataRecordFormat; import org.neo4j.kernel.impl.store.format.standard.Standard; -import org.neo4j.kernel.impl.store.format.standard.StandardV2_0; -import org.neo4j.kernel.impl.store.format.standard.StandardV2_1; -import org.neo4j.kernel.impl.store.format.standard.StandardV2_2; import org.neo4j.kernel.impl.store.format.standard.StandardV2_3; import org.neo4j.kernel.impl.storemigration.StoreMigrationParticipant; import org.neo4j.kernel.impl.storemigration.StoreUpgrader; @@ -68,7 +63,6 @@ import org.neo4j.kernel.impl.storemigration.UpgradableDatabase; import org.neo4j.kernel.impl.storemigration.UpgradeNotAllowedByConfigurationException; import org.neo4j.kernel.impl.storemigration.legacylogs.LegacyLogFilenames; -import org.neo4j.kernel.impl.storemigration.legacystore.LegacyStoreVersionCheck; import org.neo4j.kernel.impl.storemigration.monitoring.MigrationProgressMonitor; import org.neo4j.kernel.impl.storemigration.monitoring.SilentMigrationProgressMonitor; import org.neo4j.kernel.impl.storemigration.participant.AbstractStoreMigrationParticipant; @@ -76,7 +70,6 @@ import org.neo4j.kernel.impl.storemigration.participant.StoreMigrator; import org.neo4j.kernel.monitoring.Monitors; import org.neo4j.logging.NullLogProvider; -import org.neo4j.test.TestGraphDatabaseFactory; import org.neo4j.test.rule.NeoStoreDataSourceRule; import org.neo4j.test.rule.PageCacheRule; import org.neo4j.test.rule.TestDirectory; @@ -91,26 +84,16 @@ import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.not; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; -import static org.junit.Assume.assumeTrue; import static org.junit.internal.matchers.ThrowableMessageMatcher.hasMessage; import static org.mockito.Matchers.any; import static org.mockito.Matchers.eq; import static org.mockito.Mockito.verify; -import static org.neo4j.consistency.store.StoreAssertions.assertConsistentStore; -import static org.neo4j.kernel.impl.storemigration.MigrationTestUtils.allLegacyStoreFilesHaveVersion; -import static org.neo4j.kernel.impl.storemigration.MigrationTestUtils.allStoreFilesHaveNoTrailer; -import static org.neo4j.kernel.impl.storemigration.MigrationTestUtils.changeVersionNumber; -import static org.neo4j.kernel.impl.storemigration.MigrationTestUtils.containsAnyStoreFiles; -import static org.neo4j.kernel.impl.storemigration.MigrationTestUtils.isolatedMigrationDirectoryOf; import static org.neo4j.kernel.impl.storemigration.MigrationTestUtils.prepareSampleLegacyDatabase; import static org.neo4j.kernel.impl.storemigration.MigrationTestUtils.removeCheckPointFromTxLog; -import static org.neo4j.kernel.impl.storemigration.MigrationTestUtils.truncateAllFiles; -import static org.neo4j.kernel.impl.storemigration.MigrationTestUtils.truncateFile; import static org.neo4j.kernel.impl.storemigration.MigrationTestUtils.verifyFilesHaveSameContent; @RunWith( Parameterized.class ) @@ -129,7 +112,6 @@ public class StoreUpgraderTest private File dbDirectory; private final FileSystemAbstraction fileSystem = fileSystemRule.get(); - private StoreVersionCheck check; private final String version; private final SchemaIndexProvider schemaIndexProvider = new InMemoryIndexProvider(); private LabelScanStoreProvider labelScanStoreProvider; @@ -146,9 +128,6 @@ public StoreUpgraderTest( String version ) public static Collection versions() { return Arrays.asList( - StandardV2_0.STORE_VERSION, - StandardV2_1.STORE_VERSION, - StandardV2_2.STORE_VERSION, StandardV2_3.STORE_VERSION ); } @@ -189,7 +168,7 @@ public File[] listFiles( File directory, FilenameFilter filter ) MetaDataStore.Position.LAST_TRANSACTION_COMMIT_TIMESTAMP, MetaDataRecordFormat.FIELD_NOT_PRESENT ); UpgradableDatabase upgradableDatabase = new UpgradableDatabase( fs, new StoreVersionCheck( pageCache ), - new LegacyStoreVersionCheck( fs ), getRecordFormats() ) + getRecordFormats() ) { @Override public RecordFormats checkUpgradeable( File storeDirectory ) @@ -199,8 +178,9 @@ public RecordFormats checkUpgradeable( File storeDirectory ) }; SilentMigrationProgressMonitor progressMonitor = new SilentMigrationProgressMonitor(); - StoreMigrator defaultMigrator = new StoreMigrator( fs, pageCache, getTuningConfig(), NullLogService.getInstance(), - schemaIndexProvider ); + StoreMigrator defaultMigrator = new StoreMigrator( fs, pageCache, getTuningConfig(), NullLogService.getInstance() + + ); StoreUpgrader upgrader = new StoreUpgrader( upgradableDatabase, progressMonitor, allowMigrateConfig, fs, pageCache, NullLogProvider.getInstance() ); upgrader.addParticipant( defaultMigrator ); @@ -212,36 +192,6 @@ public RecordFormats checkUpgradeable( File storeDirectory ) upgrader.migrateIfNeeded( storeDirectory ); } - @Test - public void shouldUpgradeAnOldFormatStore() throws IOException, ConsistencyCheckIncompleteException - { - // Given - PageCache pageCache = pageCacheRule.getPageCache( fileSystem ); - UpgradableDatabase upgradableDatabase = new UpgradableDatabase( fileSystem, - new StoreVersionCheck( pageCache ), new LegacyStoreVersionCheck( fileSystem ), - getRecordFormats() ); - Set versionSet = versionSet( StandardV2_0.STORE_VERSION, - StandardV2_1.STORE_VERSION, - StandardV2_2.STORE_VERSION ); - - assertEquals( versionSet.contains( version ), - allLegacyStoreFilesHaveVersion( fileSystem, dbDirectory, version ) ); - - // When - newUpgrader( upgradableDatabase, pageCache ).migrateIfNeeded( dbDirectory ); - - // Then - assertCorrectStoreVersion( getRecordFormats().storeVersion(), check, dbDirectory ); - assertTrue( allStoreFilesHaveNoTrailer( fileSystem, dbDirectory ) ); - - // We leave logical logs in place since the new version can read the old - - assertFalse( containsAnyStoreFiles( fileSystem, isolatedMigrationDirectoryOf( dbDirectory ) ) ); - // Since consistency checker is in read only mode we need to start/stop db to generate label scan store. - startStopDatabase(); - assertConsistentStore( dbDirectory ); - } - @Test public void shouldHaltUpgradeIfUpgradeConfigurationVetoesTheProcess() throws IOException { @@ -250,7 +200,7 @@ public void shouldHaltUpgradeIfUpgradeConfigurationVetoesTheProcess() throws IOE .allow_store_upgrade.name(), "false" ) ); UpgradableDatabase upgradableDatabase = new UpgradableDatabase( fileSystem, - new StoreVersionCheck( pageCache ), new LegacyStoreVersionCheck( fileSystem ), + new StoreVersionCheck( pageCache ), getRecordFormats() ); try @@ -264,50 +214,18 @@ public void shouldHaltUpgradeIfUpgradeConfigurationVetoesTheProcess() throws IOE } } - @Test - public void shouldLeaveAllFilesUntouchedIfWrongVersionNumberFound() - throws IOException - { - Set applicableVersions = - versionSet( StandardV2_0.STORE_VERSION, StandardV2_1.STORE_VERSION, StandardV2_2.STORE_VERSION ); - assumeTrue( "Applicable only to specified version set.", applicableVersions.contains( version ) ); - - File comparisonDirectory = new File( "target/" + StoreUpgraderTest.class.getSimpleName() - + "shouldLeaveAllFilesUntouchedIfWrongVersionNumberFound-comparison" ); - - changeVersionNumber( fileSystem, new File( dbDirectory, "neostore.nodestore.db" ), "v0.9.5" ); - fileSystem.deleteRecursively( comparisonDirectory ); - fileSystem.copyRecursively( dbDirectory, comparisonDirectory ); - PageCache pageCache = pageCacheRule.getPageCache( fileSystem ); - UpgradableDatabase upgradableDatabase = new UpgradableDatabase( fileSystem, - new StoreVersionCheck( pageCache ), new LegacyStoreVersionCheck( fileSystem ), - getRecordFormats() ); - - try - { - newUpgrader( upgradableDatabase, pageCache ).migrateIfNeeded( dbDirectory ); - fail( "Should throw exception" ); - } - catch ( StoreUpgrader.UnexpectedUpgradingStoreVersionException e ) - { - // expected - } - - verifyFilesHaveSameContent( fileSystem, comparisonDirectory, dbDirectory ); - } - @Test public void shouldRefuseToUpgradeIfAnyOfTheStoresWereNotShutDownCleanly() throws IOException { File comparisonDirectory = new File( "target/" + StoreUpgraderTest.class.getSimpleName() + "shouldRefuseToUpgradeIfAnyOfTheStoresWereNotShutDownCleanly-comparison" ); - makeDbNotCleanlyShutdown( false ); + removeCheckPointFromTxLog( fileSystem, dbDirectory ); fileSystem.deleteRecursively( comparisonDirectory ); fileSystem.copyRecursively( dbDirectory, comparisonDirectory ); PageCache pageCache = pageCacheRule.getPageCache( fileSystem ); UpgradableDatabase upgradableDatabase = new UpgradableDatabase( fileSystem, - new StoreVersionCheck( pageCache ), new LegacyStoreVersionCheck( fileSystem ), + new StoreVersionCheck( pageCache ), getRecordFormats() ); try @@ -329,12 +247,12 @@ public void shouldRefuseToUpgradeIfAllOfTheStoresWereNotShutDownCleanly() { File comparisonDirectory = new File( "target/" + StoreUpgraderTest.class.getSimpleName() + "shouldRefuseToUpgradeIfAllOfTheStoresWereNotShutDownCleanly-comparison" ); - makeDbNotCleanlyShutdown( true ); + removeCheckPointFromTxLog( fileSystem, dbDirectory ); fileSystem.deleteRecursively( comparisonDirectory ); fileSystem.copyRecursively( dbDirectory, comparisonDirectory ); PageCache pageCache = pageCacheRule.getPageCache( fileSystem ); UpgradableDatabase upgradableDatabase = new UpgradableDatabase( fileSystem, - new StoreVersionCheck( pageCache ), new LegacyStoreVersionCheck( fileSystem ), + new StoreVersionCheck( pageCache ), getRecordFormats() ); try @@ -355,7 +273,7 @@ public void shouldContinueMovingFilesIfUpgradeCancelledWhileMoving() throws Exce { PageCache pageCache = pageCacheRule.getPageCache( fileSystem ); UpgradableDatabase upgradableDatabase = new UpgradableDatabase( fileSystem, - new StoreVersionCheck( pageCache ), new LegacyStoreVersionCheck( fileSystem ), + new StoreVersionCheck( pageCache ), getRecordFormats() ); String versionToMigrateTo = upgradableDatabase.currentVersion(); @@ -406,7 +324,7 @@ public void upgradedNeoStoreShouldHaveNewUpgradeTimeAndUpgradeId() throws Except fileSystem.deleteFile( new File( dbDirectory, StoreLogService.INTERNAL_LOG_NAME ) ); PageCache pageCache = pageCacheRule.getPageCache( fileSystem ); UpgradableDatabase upgradableDatabase = new UpgradableDatabase( fileSystem, - new StoreVersionCheck( pageCache ), new LegacyStoreVersionCheck( fileSystem ), + new StoreVersionCheck( pageCache ), getRecordFormats() ); // When @@ -433,7 +351,7 @@ public void upgradeShouldNotLeaveLeftoverAndMigrationDirs() throws Exception fileSystem.deleteFile( new File( dbDirectory, StoreLogService.INTERNAL_LOG_NAME ) ); PageCache pageCache = pageCacheRule.getPageCache( fileSystem ); UpgradableDatabase upgradableDatabase = new UpgradableDatabase( fileSystem, - new StoreVersionCheck( pageCache ), new LegacyStoreVersionCheck( fileSystem ), + new StoreVersionCheck( pageCache ), getRecordFormats() ); // When @@ -457,7 +375,7 @@ public void upgraderShouldCleanupLegacyLeftoverAndMigrationDirs() throws Excepti // When UpgradableDatabase upgradableDatabase = new UpgradableDatabase( fileSystem, - new StoreVersionCheck( pageCache ), new LegacyStoreVersionCheck( fileSystem ), + new StoreVersionCheck( pageCache ), getRecordFormats() ); StoreUpgrader storeUpgrader = newUpgrader( upgradableDatabase, pageCache ); storeUpgrader.migrateIfNeeded( dbDirectory ); @@ -472,13 +390,6 @@ protected void prepareSampleDatabase( String version, FileSystemAbstraction file prepareSampleLegacyDatabase( version, fileSystem, dbDirectory, databaseDirectory ); } - private static void assertCorrectStoreVersion( String expectedStoreVersion, StoreVersionCheck check, File storeDir ) - { - File neoStoreFile = new File( storeDir, MetaDataStore.DEFAULT_NAME ); - StoreVersionCheck.Result result = check.hasVersion( neoStoreFile, expectedStoreVersion ); - assertTrue( "Unexpected store version", result.outcome.isSuccessful() ); - } - private StoreMigrationParticipant participantThatWillFailWhenMoving( final String failureMessage ) { return new AbstractStoreMigrationParticipant( "Failing" ) @@ -506,12 +417,11 @@ private StoreUpgrader newUpgrader( UpgradableDatabase upgradableDatabase, PageCa private StoreUpgrader newUpgrader( UpgradableDatabase upgradableDatabase, PageCache pageCache, Config config ) throws IOException { - check = new StoreVersionCheck( pageCache ); SilentMigrationProgressMonitor progressMonitor = new SilentMigrationProgressMonitor(); NullLogService instance = NullLogService.getInstance(); - StoreMigrator defaultMigrator = new StoreMigrator( fileSystem, pageCache, getTuningConfig(), instance, - schemaIndexProvider ); + StoreMigrator defaultMigrator = new StoreMigrator( fileSystem, pageCache, getTuningConfig(), instance + ); SchemaIndexMigrator indexMigrator = new SchemaIndexMigrator( fileSystem, schemaIndexProvider, labelScanStoreProvider ); @@ -536,29 +446,6 @@ private List migrationHelperDirs() return Arrays.asList( tmpDirs ); } - private void makeDbNotCleanlyShutdown( boolean truncateAll ) throws IOException - { - Set truncateVersions = versionSet( StandardV2_0.STORE_VERSION, StandardV2_1.STORE_VERSION, - StandardV2_2.STORE_VERSION ); - if ( truncateVersions.contains( version ) ) - { - if ( truncateAll ) - { - truncateAllFiles( fileSystem, dbDirectory, version ); - - } - else - { - File storeFile = new File( dbDirectory, "neostore.propertystore.db.index.keys" ); - truncateFile( fileSystem, storeFile, "StringPropertyStore " + version ); - } - } - else - { - removeCheckPointFromTxLog( fileSystem, dbDirectory ); - } - } - private Config getTuningConfig() { return Config.embeddedDefaults( MapUtil.stringMap( GraphDatabaseSettings.record_format.name(), getRecordFormatsName() ) ); @@ -578,10 +465,4 @@ private Set versionSet( String... versions ) { return Stream.of( versions ).collect( Collectors.toSet() ); } - - private void startStopDatabase() - { - GraphDatabaseService databaseService = new TestGraphDatabaseFactory().newEmbeddedDatabase( dbDirectory ); - databaseService.shutdown(); - } } diff --git a/enterprise/ha/src/test/java/org/neo4j/kernel/ha/SlaveUpgradeTest.java b/enterprise/ha/src/test/java/org/neo4j/kernel/ha/SlaveUpgradeTest.java index 1ae22239192c..9fcf9121b763 100644 --- a/enterprise/ha/src/test/java/org/neo4j/kernel/ha/SlaveUpgradeTest.java +++ b/enterprise/ha/src/test/java/org/neo4j/kernel/ha/SlaveUpgradeTest.java @@ -46,7 +46,7 @@ public void haShouldFailToStartWithOldStore() throws Exception try { File dir = testDirectory.directory( "haShouldFailToStartWithOldStore" ); - MigrationTestUtils.find20FormatStoreDirectory( dir ); + MigrationTestUtils.find23FormatStoreDirectory( dir ); new TestHighlyAvailableGraphDatabaseFactory() .newEmbeddedDatabaseBuilder( dir ) diff --git a/enterprise/kernel/src/test/java/org/neo4j/kernel/impl/store/format/RecordFormatSelectorTest.java b/enterprise/kernel/src/test/java/org/neo4j/kernel/impl/store/format/RecordFormatSelectorTest.java index 520a6956bd6d..7ab50dc98c10 100644 --- a/enterprise/kernel/src/test/java/org/neo4j/kernel/impl/store/format/RecordFormatSelectorTest.java +++ b/enterprise/kernel/src/test/java/org/neo4j/kernel/impl/store/format/RecordFormatSelectorTest.java @@ -91,9 +91,6 @@ public void defaultFormatTest() @Test public void selectForVersionTest() { - assertSame( StandardV2_0.RECORD_FORMATS, selectForVersion( StandardV2_0.STORE_VERSION ) ); - assertSame( StandardV2_1.RECORD_FORMATS, selectForVersion( StandardV2_1.STORE_VERSION ) ); - assertSame( StandardV2_2.RECORD_FORMATS, selectForVersion( StandardV2_2.STORE_VERSION ) ); assertSame( StandardV2_3.RECORD_FORMATS, selectForVersion( StandardV2_3.STORE_VERSION ) ); assertSame( StandardV3_0.RECORD_FORMATS, selectForVersion( StandardV3_0.STORE_VERSION ) ); assertSame( StandardV3_2.RECORD_FORMATS, selectForVersion( StandardV3_2.STORE_VERSION ) ); @@ -147,9 +144,6 @@ public void selectForConfigWithWrongRecordFormatParameter() public void selectForStoreWithValidStore() throws IOException { PageCache pageCache = getPageCache(); - verifySelectForStore( pageCache, StandardV2_0.RECORD_FORMATS ); - verifySelectForStore( pageCache, StandardV2_1.RECORD_FORMATS ); - verifySelectForStore( pageCache, StandardV2_2.RECORD_FORMATS ); verifySelectForStore( pageCache, StandardV2_3.RECORD_FORMATS ); verifySelectForStore( pageCache, StandardV3_0.RECORD_FORMATS ); verifySelectForStore( pageCache, HighLimitV3_0_0.RECORD_FORMATS ); diff --git a/enterprise/kernel/src/test/java/org/neo4j/kernel/impl/store/format/highlimit/HighLimitStoreMigrationTest.java b/enterprise/kernel/src/test/java/org/neo4j/kernel/impl/store/format/highlimit/HighLimitStoreMigrationTest.java index 3fcd44f523b1..d1afc059a6a8 100644 --- a/enterprise/kernel/src/test/java/org/neo4j/kernel/impl/store/format/highlimit/HighLimitStoreMigrationTest.java +++ b/enterprise/kernel/src/test/java/org/neo4j/kernel/impl/store/format/highlimit/HighLimitStoreMigrationTest.java @@ -45,7 +45,6 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertThat; import static org.mockito.Mockito.mock; - import static org.neo4j.kernel.impl.store.MetaDataStore.Position.STORE_VERSION; public class HighLimitStoreMigrationTest @@ -71,8 +70,8 @@ public void migrateHighLimit3_0StoreFiles() throws IOException PageCache pageCache = pageCacheRule.getPageCache( fileSystem ); SchemaIndexProvider schemaIndexProvider = mock( SchemaIndexProvider.class ); - StoreMigrator migrator = new StoreMigrator( fileSystem, pageCache, Config.empty(), NullLogService.getInstance(), - schemaIndexProvider ); + StoreMigrator migrator = new StoreMigrator( fileSystem, pageCache, Config.empty(), NullLogService.getInstance() + ); File storeDir = new File( testDirectory.graphDbDir(), "storeDir" ); File migrationDir = new File( testDirectory.graphDbDir(), "migrationDir" ); diff --git a/enterprise/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/StoreMigrationIT.java b/enterprise/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/StoreMigrationIT.java index 9d4ea50fe319..9ad23a7400ed 100644 --- a/enterprise/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/StoreMigrationIT.java +++ b/enterprise/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/StoreMigrationIT.java @@ -55,7 +55,6 @@ import org.neo4j.kernel.impl.store.format.standard.StandardV2_3; import org.neo4j.kernel.impl.store.format.standard.StandardV3_0; import org.neo4j.kernel.impl.store.format.standard.StandardV3_2; -import org.neo4j.kernel.impl.storemigration.legacystore.LegacyStoreVersionCheck; import org.neo4j.logging.NullLogProvider; import org.neo4j.test.rule.PageCacheRule; import org.neo4j.test.rule.TestDirectory; @@ -103,14 +102,12 @@ public static Iterable data() throws IOException PageCache pageCache = pageCacheRule.getPageCache( fs ); File dir = TestDirectory.testDirectory( StoreMigrationIT.class ).prepareDirectoryForTest( "migration" ); StoreVersionCheck storeVersionCheck = new StoreVersionCheck( pageCache ); - LegacyStoreVersionCheck legacyStoreVersionCheck = new LegacyStoreVersionCheck( fs ); List data = new ArrayList<>(); ArrayList recordFormatses = new ArrayList<>(); RecordFormatSelector.allFormats().forEach( ( f ) -> addIfNotThere( f, recordFormatses ) ); for ( RecordFormats toFormat : recordFormatses ) { - UpgradableDatabase upgradableDatabase = new UpgradableDatabase( fs, storeVersionCheck, - legacyStoreVersionCheck, toFormat ); + UpgradableDatabase upgradableDatabase = new UpgradableDatabase( fs, storeVersionCheck, toFormat ); for ( RecordFormats fromFormat : recordFormatses ) { File db = new File( dir, baseDirName( toFormat, fromFormat ) ); diff --git a/enterprise/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/UpgradableDatabaseTest.java b/enterprise/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/UpgradableDatabaseTest.java index c0923d811e5a..53db8ff18674 100644 --- a/enterprise/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/UpgradableDatabaseTest.java +++ b/enterprise/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/UpgradableDatabaseTest.java @@ -32,6 +32,7 @@ import java.io.IOException; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.io.fs.FileSystemAbstraction; @@ -41,13 +42,8 @@ import org.neo4j.kernel.impl.store.format.StoreVersion; import org.neo4j.kernel.impl.store.format.standard.Standard; import org.neo4j.kernel.impl.store.format.standard.StandardFormatFamily; -import org.neo4j.kernel.impl.store.format.standard.StandardV2_0; -import org.neo4j.kernel.impl.store.format.standard.StandardV2_1; -import org.neo4j.kernel.impl.store.format.standard.StandardV2_2; import org.neo4j.kernel.impl.store.format.standard.StandardV2_3; -import org.neo4j.kernel.impl.storemigration.legacystore.LegacyStoreVersionCheck; import org.neo4j.kernel.internal.Version; -import org.neo4j.string.UTF8; import org.neo4j.test.rule.PageCacheRule; import org.neo4j.test.rule.TestDirectory; import org.neo4j.test.rule.fs.DefaultFileSystemRule; @@ -60,7 +56,6 @@ import static org.neo4j.kernel.impl.store.MetaDataStore.Position.STORE_VERSION; import static org.neo4j.kernel.impl.storemigration.MigrationTestUtils.changeVersionNumber; import static org.neo4j.kernel.impl.storemigration.MigrationTestUtils.removeCheckPointFromTxLog; -import static org.neo4j.kernel.impl.storemigration.MigrationTestUtils.truncateToFixedLength; import static org.neo4j.kernel.impl.storemigration.StoreUpgrader.UnexpectedUpgradingStoreVersionException.MESSAGE; @RunWith( Enclosed.class ) @@ -87,10 +82,7 @@ public static class SupportedVersions @Parameterized.Parameters( name = "{0}" ) public static Collection versions() { - return Arrays.asList( - StandardV2_0.STORE_VERSION, - StandardV2_1.STORE_VERSION, - StandardV2_2.STORE_VERSION, + return Collections.singletonList( StandardV2_3.STORE_VERSION ); } @@ -121,8 +113,7 @@ public void shouldAcceptTheStoresInTheSampleDatabaseAsBeingEligibleForUpgrade() { // given final UpgradableDatabase upgradableDatabase = new UpgradableDatabase( fileSystem, - new StoreVersionCheck( pageCacheRule.getPageCache( fileSystem ) ), - new LegacyStoreVersionCheck( fileSystem ), getRecordFormat() ); + new StoreVersionCheck( pageCacheRule.getPageCache( fileSystem ) ), getRecordFormat() ); // when final boolean result = storeFilesUpgradeable( workingDirectory, upgradableDatabase ); @@ -131,32 +122,12 @@ public void shouldAcceptTheStoresInTheSampleDatabaseAsBeingEligibleForUpgrade() assertTrue( result ); } - @Test - public void shouldRejectStoresIfOneFileHasIncorrectVersion() throws IOException - { - // there are no store trailers in 2.3 - Assume.assumeFalse( StandardV2_3.STORE_VERSION.equals( version ) ); - - // given - changeVersionNumber( fileSystem, new File( workingDirectory, "neostore.nodestore.db" ), "v0.9.5" ); - final UpgradableDatabase upgradableDatabase = new UpgradableDatabase( fileSystem, - new StoreVersionCheck( pageCacheRule.getPageCache( fileSystem ) ), - new LegacyStoreVersionCheck( fileSystem ), getRecordFormat() ); - - // when - final boolean result = storeFilesUpgradeable( workingDirectory, upgradableDatabase ); - - // then - assertFalse( result ); - } - @Test public void shouldDetectOldVersionAsDifferentFromCurrent() throws Exception { // given final UpgradableDatabase upgradableDatabase = new UpgradableDatabase( fileSystem, - new StoreVersionCheck( pageCacheRule.getPageCache( fileSystem ) ), - new LegacyStoreVersionCheck( fileSystem ), getRecordFormat() ); + new StoreVersionCheck( pageCacheRule.getPageCache( fileSystem ) ), getRecordFormat() ); // when boolean currentVersion = upgradableDatabase.hasCurrentVersion( workingDirectory ); @@ -165,28 +136,6 @@ public void shouldDetectOldVersionAsDifferentFromCurrent() throws Exception assertFalse( currentVersion ); } - @Test - public void shouldRejectStoresIfOneFileShorterThanExpectedVersionString() throws IOException - { - // there are no store trailers in 2.3 - Assume.assumeFalse( StandardV2_3.STORE_VERSION.equals( version ) ); - - // given - final int shortFileLength = 5 /* (RelationshipTypeStore.RECORD_SIZE) */ * 3; - assertTrue( shortFileLength < UTF8.encode( "StringPropertyStore " + version ).length ); - truncateToFixedLength( fileSystem, new File( workingDirectory, "neostore.relationshiptypestore.db" ), - shortFileLength ); - final UpgradableDatabase upgradableDatabase = new UpgradableDatabase( fileSystem, - new StoreVersionCheck( pageCacheRule.getPageCache( fileSystem ) ), - new LegacyStoreVersionCheck( fileSystem ), getRecordFormat() ); - - // when - final boolean result = storeFilesUpgradeable( workingDirectory, upgradableDatabase ); - - // then - assertFalse( result ); - } - @Test public void shouldRejectStoresIfDBIsNotShutdownCleanly() throws IOException { @@ -196,8 +145,7 @@ public void shouldRejectStoresIfDBIsNotShutdownCleanly() throws IOException // given removeCheckPointFromTxLog( fileSystem, workingDirectory ); final UpgradableDatabase upgradableDatabase = new UpgradableDatabase( fileSystem, - new StoreVersionCheck( pageCacheRule.getPageCache( fileSystem ) ), - new LegacyStoreVersionCheck( fileSystem ), getRecordFormat() ); + new StoreVersionCheck( pageCacheRule.getPageCache( fileSystem ) ), getRecordFormat() ); // when final boolean result = storeFilesUpgradeable( workingDirectory, upgradableDatabase ); @@ -238,7 +186,7 @@ public void setup() throws IOException fileSystem = fileSystemRule.get(); workingDirectory = testDirectory.graphDbDir(); // doesn't matter which version we pick we are changing it to the wrong one... - MigrationTestUtils.findFormatStoreDirectoryForVersion( StandardV2_1.STORE_VERSION, workingDirectory ); + MigrationTestUtils.findFormatStoreDirectoryForVersion( StandardV2_3.STORE_VERSION, workingDirectory ); changeVersionNumber( fileSystem, new File( workingDirectory, neostoreFilename ), version ); File metadataStore = new File( workingDirectory, MetaDataStore.DEFAULT_NAME ); MetaDataStore.setRecord( pageCacheRule.getPageCache( fileSystem ), metadataStore, STORE_VERSION, @@ -250,8 +198,7 @@ public void shouldDetectOldVersionAsDifferentFromCurrent() throws Exception { // given final UpgradableDatabase upgradableDatabase = new UpgradableDatabase( fileSystem, - new StoreVersionCheck( pageCacheRule.getPageCache( fileSystem ) ), - new LegacyStoreVersionCheck( fileSystem ), getRecordFormat() ); + new StoreVersionCheck( pageCacheRule.getPageCache( fileSystem ) ), getRecordFormat() ); // when boolean currentVersion = upgradableDatabase.hasCurrentVersion( workingDirectory ); @@ -265,8 +212,7 @@ public void shouldCommunicateWhatCausesInabilityToUpgrade() { // given final UpgradableDatabase upgradableDatabase = new UpgradableDatabase( fileSystem, - new StoreVersionCheck( pageCacheRule.getPageCache( fileSystem ) ), - new LegacyStoreVersionCheck( fileSystem ), getRecordFormat() ); + new StoreVersionCheck( pageCacheRule.getPageCache( fileSystem ) ), getRecordFormat() ); try { // when diff --git a/enterprise/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigratorTest.java b/enterprise/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigratorTest.java index fd4752e6d95e..56210549511d 100644 --- a/enterprise/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigratorTest.java +++ b/enterprise/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigratorTest.java @@ -47,7 +47,6 @@ import static org.neo4j.graphdb.factory.GraphDatabaseSettings.pagecache_memory; import static org.neo4j.helpers.collection.MapUtil.stringMap; import static org.neo4j.io.pagecache.tracing.PageCacheTracer.NULL; -import static org.neo4j.kernel.api.index.SchemaIndexProvider.NO_INDEX_PROVIDER; import static org.neo4j.kernel.impl.store.StoreFile.NEO_STORE; import static org.neo4j.kernel.impl.storemigration.StoreFileType.STORE; @@ -80,8 +79,8 @@ public void shouldNotDoActualStoreMigrationBetween3_0_5_and_next() throws Except assertTrue( hasVersionResult.actualVersion, hasVersionResult.outcome.isSuccessful() ); // WHEN - StoreMigrator migrator = new StoreMigrator( fs, pageCache, config, NullLogService.getInstance(), - NO_INDEX_PROVIDER ); + StoreMigrator migrator = new StoreMigrator( fs, pageCache, config, NullLogService.getInstance() + ); MigrationProgressMonitor.Section monitor = mock( MigrationProgressMonitor.Section.class ); File migrationDir = new File( storeDir, "migration" ); fs.mkdirs( migrationDir ); diff --git a/enterprise/neo4j-enterprise/src/test/java/org/neo4j/upgrade/StoreMigratorFrom20IT.java b/enterprise/neo4j-enterprise/src/test/java/org/neo4j/upgrade/StoreMigratorFrom20IT.java deleted file mode 100644 index 82cce714c03d..000000000000 --- a/enterprise/neo4j-enterprise/src/test/java/org/neo4j/upgrade/StoreMigratorFrom20IT.java +++ /dev/null @@ -1,273 +0,0 @@ -/* - * Copyright (c) 2002-2017 "Neo Technology," - * Network Engine for Objects in Lund AB [http://neotechnology.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -package org.neo4j.upgrade; - -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.RuleChain; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; -import org.junit.runners.Parameterized.Parameter; -import org.junit.runners.Parameterized.Parameters; -import upgrade.DatabaseContentVerifier; -import upgrade.ListAccumulatorMigrationProgressMonitor; - -import java.io.File; -import java.io.IOException; -import java.util.Arrays; -import java.util.List; -import java.util.Map; - -import org.neo4j.consistency.checking.full.ConsistencyCheckIncompleteException; -import org.neo4j.graphdb.GraphDatabaseService; -import org.neo4j.graphdb.Transaction; -import org.neo4j.graphdb.factory.EnterpriseGraphDatabaseFactory; -import org.neo4j.graphdb.factory.GraphDatabaseSettings; -import org.neo4j.io.fs.FileSystemAbstraction; -import org.neo4j.io.pagecache.PageCache; -import org.neo4j.kernel.api.impl.index.storage.DirectoryFactory; -import org.neo4j.kernel.api.impl.schema.LuceneSchemaIndexProvider; -import org.neo4j.kernel.api.index.SchemaIndexProvider; -import org.neo4j.kernel.configuration.Config; -import org.neo4j.kernel.ha.HaSettings; -import org.neo4j.kernel.ha.HighlyAvailableGraphDatabase; -import org.neo4j.kernel.impl.MyRelTypes; -import org.neo4j.kernel.impl.api.scan.LabelScanStoreProvider; -import org.neo4j.kernel.impl.factory.OperationalMode; -import org.neo4j.kernel.impl.ha.ClusterManager; -import org.neo4j.kernel.impl.logging.NullLogService; -import org.neo4j.kernel.impl.store.MetaDataStore; -import org.neo4j.kernel.impl.store.NeoStores; -import org.neo4j.kernel.impl.store.StoreFactory; -import org.neo4j.kernel.impl.store.format.RecordFormats; -import org.neo4j.kernel.impl.store.format.highlimit.HighLimit; -import org.neo4j.kernel.impl.store.format.standard.Standard; -import org.neo4j.kernel.impl.storemigration.StoreUpgrader; -import org.neo4j.kernel.impl.storemigration.StoreVersionCheck; -import org.neo4j.kernel.impl.storemigration.UpgradableDatabase; -import org.neo4j.kernel.impl.storemigration.legacystore.LegacyStoreVersionCheck; -import org.neo4j.kernel.impl.storemigration.participant.SchemaIndexMigrator; -import org.neo4j.kernel.impl.storemigration.participant.StoreMigrator; -import org.neo4j.kernel.lifecycle.LifeSupport; -import org.neo4j.kernel.monitoring.Monitors; -import org.neo4j.logging.LogProvider; -import org.neo4j.logging.NullLogProvider; -import org.neo4j.test.rule.NeoStoreDataSourceRule; -import org.neo4j.test.rule.PageCacheRule; -import org.neo4j.test.rule.TestDirectory; -import org.neo4j.test.rule.fs.DefaultFileSystemRule; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.neo4j.consistency.store.StoreAssertions.assertConsistentStore; -import static org.neo4j.helpers.collection.MapUtil.stringMap; -import static org.neo4j.kernel.impl.ha.ClusterManager.allSeesAllAsAvailable; -import static org.neo4j.kernel.impl.storemigration.MigrationTestUtils.find20FormatStoreDirectory; -import static org.neo4j.upgrade.StoreMigratorTestUtil.buildClusterWithMasterDirIn; - -@RunWith( Parameterized.class ) -public class StoreMigratorFrom20IT -{ - private final TestDirectory storeDir = TestDirectory.testDirectory(); - private final PageCacheRule pageCacheRule = new PageCacheRule(); - private final DefaultFileSystemRule fileSystemRule = new DefaultFileSystemRule(); - - @Rule - public RuleChain ruleChain = RuleChain.outerRule( storeDir ) - .around( fileSystemRule ).around( pageCacheRule ); - - private final ListAccumulatorMigrationProgressMonitor monitor = new ListAccumulatorMigrationProgressMonitor(); - private FileSystemAbstraction fs; - private PageCache pageCache; - private final LifeSupport life = new LifeSupport(); - private UpgradableDatabase upgradableDatabase; - private SchemaIndexProvider schemaIndexProvider; - private LabelScanStoreProvider labelScanStoreProvider; - - @Parameter - public String recordFormatName; - @Parameter( 1 ) - public RecordFormats recordFormat; - - @Parameters( name = "{0}" ) - public static List recordFormats() - { - return Arrays.asList( - new Object[]{Standard.LATEST_NAME, Standard.LATEST_RECORD_FORMATS}, - new Object[]{HighLimit.NAME, HighLimit.RECORD_FORMATS} ); - } - - @Before - public void setUp() - { - fs = fileSystemRule.get(); - pageCache = pageCacheRule.getPageCache( fs ); - - File storeDirectory = storeDir.directory(); - schemaIndexProvider = new LuceneSchemaIndexProvider( fs, DirectoryFactory.PERSISTENT, storeDirectory, - NullLogProvider.getInstance(), Config.empty(), OperationalMode.single ); - labelScanStoreProvider = NeoStoreDataSourceRule.nativeLabelScanStoreProvider( storeDirectory, fs, pageCache, - new Monitors() ); - - upgradableDatabase = new UpgradableDatabase( fs, new StoreVersionCheck( pageCache ), - new LegacyStoreVersionCheck( fs ), recordFormat ); - } - - @After - public void tearDown() - { - life.shutdown(); - } - - @Test - public void shouldMigrate() throws IOException, ConsistencyCheckIncompleteException - { - // WHEN - StoreMigrator storeMigrator = new StoreMigrator( fs, pageCache, getConfig(), NullLogService.getInstance(), - schemaIndexProvider ); - SchemaIndexMigrator indexMigrator = new SchemaIndexMigrator( fs, schemaIndexProvider, labelScanStoreProvider ); - upgrader( indexMigrator, storeMigrator ).migrateIfNeeded( find20FormatStoreDirectory( storeDir.directory() ) ); - - // THEN - assertEquals( 2, monitor.progresses().size() ); - assertTrue( monitor.isStarted() ); - assertTrue( monitor.isFinished() ); - - GraphDatabaseService database = new EnterpriseGraphDatabaseFactory() - .newEmbeddedDatabaseBuilder( storeDir.absolutePath() ) - .newGraphDatabase(); - try - { - verifyDatabaseContents( database ); - } - finally - { - // CLEANUP - database.shutdown(); - } - - LogProvider logProvider = NullLogProvider.getInstance(); - StoreFactory storeFactory = new StoreFactory( storeDir.directory(), pageCache, fs, logProvider ); - try ( NeoStores neoStores = storeFactory.openAllNeoStores( true ) ) - { - verifyNeoStore( neoStores ); - } - assertConsistentStore( storeDir.directory() ); - } - - @Test - public void shouldMigrateCluster() throws Throwable - { - // Given - File legacyStoreDir = find20FormatStoreDirectory( storeDir.directory( "legacy-indexes" ) ); - - // When - StoreMigrator storeMigrator = new StoreMigrator( fs, pageCache, getConfig(), NullLogService.getInstance(), - schemaIndexProvider ); - SchemaIndexMigrator indexMigrator = new SchemaIndexMigrator( fs, schemaIndexProvider, labelScanStoreProvider ); - upgrader( indexMigrator, storeMigrator ).migrateIfNeeded( legacyStoreDir ); - ClusterManager.ManagedCluster cluster = - buildClusterWithMasterDirIn( fs, legacyStoreDir, life, getParams() ); - cluster.await( allSeesAllAsAvailable() ); - cluster.sync(); - - // Then - HighlyAvailableGraphDatabase slave1 = cluster.getAnySlave(); - verifySlaveContents( slave1 ); - verifySlaveContents( cluster.getAnySlave( slave1 ) ); - verifyDatabaseContents( cluster.getMaster() ); - } - - private static void verifyDatabaseContents( GraphDatabaseService database ) - { - DatabaseContentVerifier verifier = new DatabaseContentVerifier( database, 2 ); - verifyNumberOfNodesAndRelationships( verifier ); - createNewNode( database ); - createNewRelationship( database ); - verifier.verifyLegacyIndex(); - verifier.verifyIndex(); - verifier.verifyJohnnyLabels(); - } - - private static void createNewNode( GraphDatabaseService database ) - { - try ( Transaction tx = database.beginTx() ) - { - database.createNode(); - tx.success(); - } - } - - private static void createNewRelationship( GraphDatabaseService database ) - { - try ( Transaction tx = database.beginTx() ) - { - database.createNode().createRelationshipTo( database.createNode(), MyRelTypes.TEST ); - tx.success(); - } - } - - private static void verifySlaveContents( HighlyAvailableGraphDatabase haDb ) - { - DatabaseContentVerifier verifier = new DatabaseContentVerifier( haDb, 2 ); - verifyNumberOfNodesAndRelationships( verifier ); - } - - private static void verifyNumberOfNodesAndRelationships( DatabaseContentVerifier verifier ) - { - verifier.verifyNodes( 502 ); - verifier.verifyRelationships( 500 ); - } - - public void verifyNeoStore( NeoStores neoStores ) - { - MetaDataStore metaDataStore = neoStores.getMetaDataStore(); - assertEquals( 1317392957120L, metaDataStore.getCreationTime() ); - assertEquals( -472309512128245482L, metaDataStore.getRandomNumber() ); - assertEquals( 5L, metaDataStore.getCurrentLogVersion() ); - assertEquals( recordFormat.storeVersion(), - MetaDataStore.versionLongToString( metaDataStore.getStoreVersion() ) ); - assertEquals( 1042L, metaDataStore.getLastCommittedTransactionId() ); - } - - private StoreUpgrader upgrader( SchemaIndexMigrator indexMigrator, StoreMigrator storeMigrator ) - { - Config config = getConfig().augment( stringMap( GraphDatabaseSettings.allow_store_upgrade.name(), "true" ) ); - StoreUpgrader upgrader = new StoreUpgrader( upgradableDatabase, monitor, config, fs, pageCache, - NullLogProvider.getInstance() ); - upgrader.addParticipant( indexMigrator ); - upgrader.addParticipant( storeMigrator ); - return upgrader; - } - - private Map getParams() - { - return stringMap( - GraphDatabaseSettings.record_format.name(), recordFormatName, - HaSettings.read_timeout.name(), "2m" ); - } - - private Config getConfig() - { - return Config.embeddedDefaults( getParams() ); - } -} diff --git a/enterprise/neo4j-enterprise/src/test/java/org/neo4j/upgrade/StoreMigratorFrom21IT.java b/enterprise/neo4j-enterprise/src/test/java/org/neo4j/upgrade/StoreMigratorFrom21IT.java deleted file mode 100644 index e5982fa6819b..000000000000 --- a/enterprise/neo4j-enterprise/src/test/java/org/neo4j/upgrade/StoreMigratorFrom21IT.java +++ /dev/null @@ -1,411 +0,0 @@ -/* - * Copyright (c) 2002-2017 "Neo Technology," - * Network Engine for Objects in Lund AB [http://neotechnology.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -package org.neo4j.upgrade; - -import org.junit.Rule; -import org.junit.Test; - -import java.io.File; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.function.Predicate; -import java.util.stream.Collectors; - -import org.neo4j.consistency.ConsistencyCheckService; -import org.neo4j.graphdb.DependencyResolver; -import org.neo4j.graphdb.GraphDatabaseService; -import org.neo4j.graphdb.PropertyContainer; -import org.neo4j.graphdb.Transaction; -import org.neo4j.graphdb.factory.GraphDatabaseBuilder; -import org.neo4j.graphdb.factory.GraphDatabaseSettings; -import org.neo4j.helpers.collection.Iterators; -import org.neo4j.helpers.collection.Pair; -import org.neo4j.helpers.progress.ProgressMonitorFactory; -import org.neo4j.kernel.api.KernelAPI; -import org.neo4j.kernel.api.KernelTransaction; -import org.neo4j.kernel.api.Statement; -import org.neo4j.kernel.api.security.AnonymousContext; -import org.neo4j.kernel.configuration.Config; -import org.neo4j.kernel.impl.storemigration.MigrationTestUtils; -import org.neo4j.kernel.internal.GraphDatabaseAPI; -import org.neo4j.logging.NullLogProvider; -import org.neo4j.test.TestGraphDatabaseFactory; -import org.neo4j.test.rule.TestDirectory; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -@SuppressWarnings( "unchecked" ) -public class StoreMigratorFrom21IT -{ - - @Rule - public final TestDirectory storeDir = TestDirectory.testDirectory(); - - @Test - public void mustMendDuplicatePropertiesWhenUpgradingFromVersion21() throws Exception - { - // The rules: - // If an index is present, all duplicates should be removed and the property set to the value in the index - // If an index is not present, the property should be set to the value of the last duplicate in the property - // chain, all duplicates except the first should be removed - // If an index is not present, the first property in the duplicate chain should be kept for the users - // benefit, moved to a special property value, `__DUPLICATE_` - // - // This is the broken store that we are upgrading: - // - // (#0:Label { keyA: "actual", keyA: "phony!", keyA: "phony!" }) - // (#1 { keyA: "actual", keyA: "actual", keyA: "actual" }) - // (#2:Label { keyA: "real1", keyA: "phony", keyA: "phony", keyD: "real2", keyD: "phony", keyD: "phony" }) - // (#3 { keyA: "real1", keyA: "phony", keyA: "phony", keyD: "real2", keyD: "phony", keyD: "phony" }) - // (#4 { keyA: "actual", keyB: "actual", keyC: "actual" }) - // (#0)-[#0:REL { keyA: "actual", keyA: "actual", keyA: "actual" }]->(#1) - // (#0)-[#1:REL { keyA: "real1", keyA: "phony", keyA: "phony", - // keyD: "real2", keyE: "phony", keyF: "phony" }]->(#1) - // (#2)-[#2:REL { keyA: "actual", keyB: "actual", keyC: "actual" }]->(#0) - // - // And this is what we want to end up with, after upgrading: - // - // (#0:Label { keyA: "actual" }) - // (#1 { keyA: "actual", __DUPLICATE_keyA: "actual" }) - // (#2:Label { keyA: "real1", keyD: "real2" }) - // (#3 { keyA: "real1", __DUPLICATE_keyA_1: "real1", __DUPLICATE_keyA_2: "real1", - // keyD: "real2", __DUPLICATE_keyD_1: "real2", __DUPLICATE_keyD_2: "real2" }) - // (#4 { keyA: "actual", keyB: "actual", keyC: "actual" }) - // (#0)-[#0:REL { keyA: "actual", __DUPLICATE_keyA: "actual" }]->(#1) - // (#0)-[#1:REL { keyA: "real1", __DUPLICATE_keyA_1: "real1", __DUPLICATE_keyA_2: "real1", - // keyD: "real2", __DUPLICATE_keyD_1: "real2", __DUPLICATE_keyD_2: "real2" }]->(#1) - // (#2)-[#2:REL { keyA: "actual", keyB: "actual", keyC: "actual" }]->(#0) - - File dir = MigrationTestUtils.find21FormatStoreDirectoryWithDuplicateProperties( storeDir.directory() ); - - TestGraphDatabaseFactory factory = new TestGraphDatabaseFactory(); - GraphDatabaseBuilder builder = factory.newEmbeddedDatabaseBuilder( dir ) - .setConfig( GraphDatabaseSettings.allow_store_upgrade, "true" ); - GraphDatabaseService database = builder.newGraphDatabase(); - database.shutdown(); - ConsistencyCheckService service = new ConsistencyCheckService(); - - ConsistencyCheckService.Result result = service.runFullConsistencyCheck( dir.getAbsoluteFile(), - Config.empty(), ProgressMonitorFactory.NONE, NullLogProvider.getInstance(), false ); - assertTrue( result.isSuccessful() ); - - database = builder.newGraphDatabase(); - // Upgrade is now completed. Verify the contents: - DependencyResolver dependencyResolver = ((GraphDatabaseAPI) database).getDependencyResolver(); - - // Verify that the properties appear correct to the outside world: - try ( Transaction ignore = database.beginTx() ) - { - verifyProperties( database.getNodeById( 0 ), - Pair.of( "keyA", new Object[] {"actual", "phony!", "phony!"} ) ); - verifyProperties( database.getNodeById( 1 ), Pair.of( "keyA", - new Object[]{"actual", "actual", "actual"} ) ); - verifyProperties( database.getNodeById( 2 ), - Pair.of( "keyA", new Object[]{ "real1", "phony", "phony"} ), - Pair.of( "keyD", new Object[]{ "real2", "phony", "phony"} )); - verifyProperties( database.getNodeById( 3 ), - Pair.of("keyA", new Object[]{"real1", "real1", "real1"}), - Pair.of("keyD", new Object[]{"real2", "real2", "real2"})); - verifyProperties( database.getNodeById( 4 ), - Pair.of("keyA", new Object[]{"actual"}), - Pair.of("keyB", new Object[]{"actual"}), - Pair.of("keyC", new Object[]{"actual"})); - verifyProperties( database.getRelationshipById( 0 ), - Pair.of( "keyA", new Object[]{"actual","actual","actual"})); - verifyProperties( database.getRelationshipById( 1 ), - Pair.of( "keyA", new Object[]{"real1", "real1", "real1"} ), - Pair.of( "keyD", new Object[]{"real2", "real2", "real2"} )); - verifyProperties( database.getRelationshipById( 2 ), - Pair.of( "keyA", new Object[]{"actual"}), - Pair.of( "keyB", new Object[]{"actual"}), - Pair.of( "keyC", new Object[]{"actual"})); - } - - // Verify that there are no two properties on the entities, that have the same key: - // (This is important because the verification above cannot tell if we have two keys with the same value) - KernelAPI kernel = dependencyResolver.resolveDependency( KernelAPI.class ); - try ( KernelTransaction tx = kernel.newTransaction( KernelTransaction.Type.implicit, AnonymousContext.read() ); - Statement statement = tx.acquireStatement() ) - { - Iterators.asUniqueSet( statement.readOperations().nodeGetPropertyKeys( 0 ) ); - Iterators.asUniqueSet( statement.readOperations().nodeGetPropertyKeys( 1 ) ); - Iterators.asUniqueSet( statement.readOperations().nodeGetPropertyKeys( 2 ) ); - Iterators.asUniqueSet( statement.readOperations().relationshipGetPropertyKeys( 0 ) ); - Iterators.asUniqueSet( statement.readOperations().relationshipGetPropertyKeys( 1 ) ); - } - - database.shutdown(); - } - - private void verifyProperties( PropertyContainer entity, Pair... expectedPropertyGroups ) - { - for ( Pair group : expectedPropertyGroups ) - { - Predicate> filter = entry -> entry.getKey().equals( group.first() ) || entry - .getKey().contains( "__DUPLICATE_" + group.first() + "_" ); - Map relevantProperties = entity.getAllProperties().entrySet().stream().filter( filter ) - .collect( Collectors.toMap( entry -> entry.getKey(), entry -> entry.getValue() ) ); - assertEquals( group.other().length, relevantProperties.size() ); - List relevantValues = new ArrayList<>( relevantProperties.values() ); - for ( Object expectedValue : group.other() ) - { - assertTrue( relevantValues.remove( expectedValue ) ); - } - assertTrue( relevantValues.size() == 0 ); - } - } -} - -/* -The test data store has been generated by running the following program in the 2.1 code base: - -v3_2 examples; - -import java.io.File; -import java.io.IOException; - -import org.neo4j.graphdb.DependencyResolver; -import org.neo4j.graphdb.DynamicLabel; -import org.neo4j.graphdb.DynamicRelationshipType; -import org.neo4j.graphdb.GraphDatabaseService; -import org.neo4j.graphdb.Label; -import org.neo4j.graphdb.Node; -import org.neo4j.graphdb.Relationship; -import org.neo4j.graphdb.Transaction; -import org.neo4j.graphdb.factory.GraphDatabaseFactory; -import org.neo4j.kernel.internal.GraphDatabaseAPI; -import org.neo4j.kernel.impl.core.Token; -import org.neo4j.kernel.impl.nioneo.store.NeoStore; -import org.neo4j.kernel.impl.nioneo.store.NodeRecord; -import org.neo4j.kernel.impl.nioneo.store.NodeStore; -import org.neo4j.kernel.impl.nioneo.store.PrimitiveRecord; -import org.neo4j.kernel.impl.nioneo.store.PropertyBlock; -import org.neo4j.kernel.impl.nioneo.store.PropertyKeyTokenStore; -import org.neo4j.kernel.impl.nioneo.store.PropertyRecord; -import org.neo4j.kernel.impl.nioneo.store.PropertyStore; -import org.neo4j.kernel.impl.nioneo.store.RelationshipRecord; -import org.neo4j.kernel.impl.nioneo.store.RelationshipStore; -import org.neo4j.kernel.impl.nioneo.xa.NeoStoreProvider; -import org.neo4j.kernel.impl.util.FileUtils; - -public class CreateStoreWithDuplicateProperties -{ - public static void main( String[] args ) throws IOException - { - String path = "21-with-duplicate-properties"; - FileUtils.deleteRecursively( new File( path ) ); - GraphDatabaseFactory factory = new GraphDatabaseFactory(); - GraphDatabaseService db = factory.newEmbeddedDatabase( path ); - GraphDatabaseAPI api = (GraphDatabaseAPI) db; - Label label = DynamicLabel.label( "Label" ); - DynamicRelationshipType relationshipType = DynamicRelationshipType.withName( "REL" ); - String actualValue = "actual"; - String phonyValue = "phony!"; - String keyA = "keyA"; - String keyB = "keyB"; - String keyC = "keyC"; - String keyD = "keyD"; - String keyE = "keyE"; - String keyF = "keyF"; - - try ( Transaction transaction = db.beginTx() ) - { - db.schema().indexFor( label ).on( keyA ).create(); - db.schema().indexFor( label ).on( keyD ).create(); - transaction.success(); - } - - long nodeIndexedSingleId; - long nodeNotIndexedSingleId; - long nodeIndexedMultiId; - long nodeNotIndexedMultiId; - long relSingleId; - long relMultiId; - try ( Transaction transaction = db.beginTx() ) - { - // Four nodes and two relationships will have duplicate properties. - // Two nodes will be in the index, the others not. - // An indexed and a non-indexed node will have multiple distinct duplicates. - // Likewise, one of the relationships will have multiple distinct duplicates. - // Another node and another relationship will be fine. - - // Indexed single duplicate - Node nodeA = db.createNode( label ); - nodeA.setProperty( keyA, actualValue ); - nodeA.setProperty( keyB, phonyValue ); - nodeA.setProperty( keyC, phonyValue ); - nodeIndexedSingleId = nodeA.getId(); - - // Non-indexed single duplicate - Node nodeB = db.createNode(); - nodeB.setProperty( keyA, actualValue ); - nodeB.setProperty( keyB, actualValue ); - nodeB.setProperty( keyC, actualValue ); - nodeNotIndexedSingleId = nodeB.getId(); - - // Single duplicate - Relationship relA = nodeA.createRelationshipTo( nodeB, relationshipType ); - relA.setProperty( keyA, actualValue ); - relA.setProperty( keyB, actualValue ); - relA.setProperty( keyC, actualValue ); - relSingleId = relA.getId(); - - // Indexed multiple duplicates - Node nodeC = db.createNode( label ); - nodeC.setProperty( keyA, "real1" ); - nodeC.setProperty( keyB, "phony" ); - nodeC.setProperty( keyC, "phony" ); - nodeC.setProperty( keyD, "real2" ); - nodeC.setProperty( keyE, "phony" ); - nodeC.setProperty( keyF, "phony" ); - nodeIndexedMultiId = nodeC.getId(); - - // Non-indexed multiple duplicate - Node nodeD = db.createNode(); - nodeD.setProperty( keyA, "real1" ); - nodeD.setProperty( keyB, "real1" ); - nodeD.setProperty( keyC, "real1" ); - nodeD.setProperty( keyD, "real2" ); - nodeD.setProperty( keyE, "real2" ); - nodeD.setProperty( keyF, "real2" ); - nodeNotIndexedMultiId = nodeD.getId(); - - // Multiple duplicates - Relationship relB = nodeA.createRelationshipTo( nodeB, relationshipType ); - relB.setProperty( keyA, "real1" ); - relB.setProperty( keyB, "real1" ); - relB.setProperty( keyC, "real1" ); - relB.setProperty( keyD, "real2" ); - relB.setProperty( keyE, "real2" ); - relB.setProperty( keyF, "real2" ); - relMultiId = relB.getId(); - - // No duplicates - Node nodeE = db.createNode(); - nodeE.setProperty( keyA, actualValue ); - nodeE.setProperty( keyB, actualValue ); - nodeE.setProperty( keyC, actualValue ); - - // No duplicates - Relationship relC = nodeD.createRelationshipTo( nodeA, relationshipType ); - relC.setProperty( keyA, actualValue ); - relC.setProperty( keyB, actualValue ); - relC.setProperty( keyC, actualValue ); - transaction.success(); - } - - // (#0:Label { keyA: "actual", keyA: "phony!", keyA: "phony!" }) - // (#1 { keyA: "actual", keyA: "actual", keyA: "actual" }) - // (#2:Label { keyA: "real1", keyA: "phony", keyA: "phony", keyD: "real2", keyE: "phony", keyF: "phony" }) - // (#3 { keyA: "real1", keyA: "phony", keyA: "phony", keyD: "real2", keyE: "phony", keyF: "phony" }) - // (#4 { keyA: "actual", keyB: "actual", keyC: "actual" }) - // (#0)-[#0:REL { keyA: "actual", keyA: "actual", keyA: "actual" }]->(#1) - // (#0)-[#1:REL { keyA: "real1", keyA: "phony", keyA: "phony", keyD: "real2", keyE: "phony", keyF: "phony" }]->(#1) - // (#2)-[#2:REL { keyA: "actual", keyB: "actual", keyC: "actual" }]->(#0) - - DependencyResolver resolver = api.getDependencyResolver(); - NeoStoreProvider neoStoreProvider = resolver.resolveDependency( NeoStoreProvider.class ); - NeoStore neoStore = neoStoreProvider.evaluate(); - PropertyKeyTokenStore propertyKeyTokenStore = neoStore.getPropertyKeyTokenStore(); - - Token tokenA = findPropertyKeyTokenFor( propertyKeyTokenStore, keyA ); - Token tokenB = findPropertyKeyTokenFor( propertyKeyTokenStore, keyB ); - Token tokenC = findPropertyKeyTokenFor( propertyKeyTokenStore, keyC ); - Token tokenD = findPropertyKeyTokenFor( propertyKeyTokenStore, keyD ); - Token tokenE = findPropertyKeyTokenFor( propertyKeyTokenStore, keyE ); - Token tokenF = findPropertyKeyTokenFor( propertyKeyTokenStore, keyF ); - - NodeStore nodeStore = neoStore.getNodeStore(); - RelationshipStore relationshipStore = neoStore.getRelationshipStore(); - PropertyStore propertyStore = neoStore.getPropertyStore(); - - NodeRecord indexedNodeSingle = nodeStore.getRecord( nodeIndexedSingleId ); - NodeRecord nonIndexedNodeSingle = nodeStore.getRecord( nodeNotIndexedSingleId ); - NodeRecord indexedNodeMulti = nodeStore.getRecord( nodeIndexedMultiId ); - NodeRecord nonindexedNodeMulti = nodeStore.getRecord( nodeNotIndexedMultiId ); - RelationshipRecord relationshipRecordSingle = relationshipStore.getRecord( relSingleId ); - RelationshipRecord relationshipRecordMulti = relationshipStore.getRecord( relMultiId ); - - replacePropertyKey( propertyStore, indexedNodeSingle, tokenB, tokenA ); - replacePropertyKey( propertyStore, indexedNodeSingle, tokenC, tokenA ); - - replacePropertyKey( propertyStore, nonIndexedNodeSingle, tokenB, tokenA ); - replacePropertyKey( propertyStore, nonIndexedNodeSingle, tokenC, tokenA ); - - replacePropertyKey( propertyStore, indexedNodeMulti, tokenB, tokenA ); - replacePropertyKey( propertyStore, indexedNodeMulti, tokenC, tokenA ); - replacePropertyKey( propertyStore, indexedNodeMulti, tokenE, tokenD ); - replacePropertyKey( propertyStore, indexedNodeMulti, tokenF, tokenD ); - - replacePropertyKey( propertyStore, nonindexedNodeMulti, tokenB, tokenA ); - replacePropertyKey( propertyStore, nonindexedNodeMulti, tokenC, tokenA ); - replacePropertyKey( propertyStore, nonindexedNodeMulti, tokenE, tokenD ); - replacePropertyKey( propertyStore, nonindexedNodeMulti, tokenF, tokenD ); - - replacePropertyKey( propertyStore, relationshipRecordSingle, tokenB, tokenA ); - replacePropertyKey( propertyStore, relationshipRecordSingle, tokenC, tokenA ); - - replacePropertyKey( propertyStore, relationshipRecordMulti, tokenB, tokenA ); - replacePropertyKey( propertyStore, relationshipRecordMulti, tokenC, tokenA ); - replacePropertyKey( propertyStore, relationshipRecordMulti, tokenE, tokenD ); - replacePropertyKey( propertyStore, relationshipRecordMulti, tokenF, tokenD ); - - db.shutdown(); - } - - private static void replacePropertyKey( - PropertyStore propertyStore, - PrimitiveRecord record, - Token original, - Token replacement ) - { - long nextProp = record.getNextProp(); - while ( nextProp != -1 ) - { - PropertyRecord propertyRecord = propertyStore.getRecord( nextProp ); - for ( PropertyBlock propertyBlock : propertyRecord.getPropertyBlocks() ) - { - if ( propertyBlock.getKeyIndexId() == original.id() ) - { - propertyBlock.setKeyIndexId( replacement.id() ); - } - } - propertyStore.updateRecord( propertyRecord ); - nextProp = propertyRecord.getNextProp(); - } - } - - private static Token findPropertyKeyTokenFor( PropertyKeyTokenStore propertyKeyTokenStore, String key ) - { - long highestPossibleIdInUse = propertyKeyTokenStore.getHighestPossibleIdInUse(); - for ( int i = 0; i <= highestPossibleIdInUse; i++ ) - { - Token token = propertyKeyTokenStore.getToken( i ); - if ( token.name().equals( key ) ) - { - return token; - } - } - return null; - } -} - */