Bump the store version and add migration path from 2.3 to 3.0

community/kernel/src/main/java/org/neo4j/kernel/NeoStoreDataSource.java

@@ -564,7 +564,7 @@ public void start() throws IOException
     private void upgradeStore( File storeDir, StoreUpgrader storeMigrationProcess, SchemaIndexProvider indexProvider )
     {
         UpgradableDatabase upgradableDatabase =
-                new UpgradableDatabase( new StoreVersionCheck( pageCache ), new LegacyStoreVersionCheck( fs ) );
+                new UpgradableDatabase( fs, new StoreVersionCheck( pageCache ), new LegacyStoreVersionCheck( fs ) );
         storeMigrationProcess
                 .addParticipant( indexProvider.storeMigrationParticipant( fs, pageCache ) );
         storeMigrationProcess.migrateIfNeeded( storeDir, upgradableDatabase, indexProvider );

community/kernel/src/main/java/org/neo4j/kernel/impl/store/CommonAbstractStore.java

@@ -51,7 +51,7 @@
  */
 public abstract class CommonAbstractStore implements IdSequence, AutoCloseable
 {
-    public static final String ALL_STORES_VERSION = "v0.A.6";
+    public static final String ALL_STORES_VERSION = "v0.A.7";
     public static final String UNKNOWN_VERSION = "Unknown";
     protected final Config configuration;
     protected final PageCache pageCache;

community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/SchemaIndexMigrator.java

@@ -44,6 +44,7 @@
 import org.neo4j.kernel.impl.storemigration.legacystore.v20.Legacy20Store;
 import org.neo4j.kernel.impl.storemigration.legacystore.v21.Legacy21Store;
 import org.neo4j.kernel.impl.storemigration.legacystore.v22.Legacy22Store;
+import org.neo4j.kernel.impl.storemigration.legacystore.v23.Legacy23Store;
 import org.neo4j.logging.NullLogProvider;
 
 import static org.neo4j.kernel.api.index.SchemaIndexProvider.getRootDirectory;
@@ -79,6 +80,7 @@ public void migrate( File storeDir, File migrationDir, SchemaIndexProvider schem
             deleteIndexesContainingArrayValues( storeDir, schemaIndexProvider );
             break;
         case Legacy22Store.LEGACY_VERSION:
+        case Legacy23Store.LEGACY_VERSION:
             break;
         default:
             throw new IllegalStateException( "Unknown version to upgrade from: " + versionToMigrateFrom );

community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/StoreMigrationTool.java

@@ -103,7 +103,7 @@ public File storeDir()
         try ( PageCache pageCache = createPageCache( fs, config ) )
         {
             UpgradableDatabase upgradableDatabase =
-                    new UpgradableDatabase( new StoreVersionCheck( pageCache ), new LegacyStoreVersionCheck( fs ) );
+                    new UpgradableDatabase( fs, new StoreVersionCheck( pageCache ), new LegacyStoreVersionCheck( fs ) );
             migrationProcess.addParticipant( new StoreMigrator(
                     new VisibleMigrationProgressMonitor( logService.getInternalLog( StoreMigrationTool.class ) ),
                     fs, pageCache, config, logService ) );

community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/StoreMigrator.java

@@ -74,6 +74,7 @@
 import org.neo4j.kernel.impl.storemigration.legacystore.v21.Legacy21Store;
 import org.neo4j.kernel.impl.storemigration.legacystore.v21.propertydeduplication.PropertyDeduplicator;
 import org.neo4j.kernel.impl.storemigration.legacystore.v22.Legacy22Store;
+import org.neo4j.kernel.impl.storemigration.legacystore.v23.Legacy23Store;
 import org.neo4j.kernel.impl.storemigration.monitoring.MigrationProgressMonitor;
 import org.neo4j.kernel.impl.transaction.log.LogPosition;
 import org.neo4j.kernel.impl.transaction.log.PhysicalLogFiles;
@@ -114,10 +115,10 @@
 
 /**
  * Migrates a neo4j kernel database from one version to the next.
- * <p/>
+ * <p>
  * Since only one store migration is supported at any given version (migration from the previous store version)
  * the migration code is specific for the current upgrade and changes with each store format version.
- * <p/>
+ * <p>
  * Just one out of many potential participants in a {@link StoreUpgrader migration}.
  *
  * @see StoreUpgrader
@@ -163,9 +164,9 @@ public void migrate( File storeDir, File migrationDir, SchemaIndexProvider schem
         writeLastTxChecksum( migrationDir, lastTxChecksum );
         writeLastTxLogPosition( migrationDir, lastTxLogPosition );
 
-
         switch ( versionToMigrateFrom )
         {
+        case Legacy23Store.LEGACY_VERSION:
         case Legacy22Store.LEGACY_VERSION:
             // nothing to do
             break;
@@ -370,7 +371,7 @@ private void migrateWithBatchImporter( File storeDir, File migrationDir, long la
 
         Configuration importConfig = new Configuration.Overridden( config );
         AdditionalInitialIds additionalInitialIds =
-                readAdditionalIds( storeDir, lastTxId, lastTxChecksum,  lastTxLogVersion, lastTxLogByteOffset );
+                readAdditionalIds( storeDir, lastTxId, lastTxChecksum, lastTxLogVersion, lastTxLogByteOffset );
         BatchImporter importer = new ParallelBatchImporter( migrationDir.getAbsoluteFile(), fileSystem,
                 importConfig, logService, withDynamicProcessorAssignment( migrationBatchImporterMonitor(
                 legacyStore ), importConfig ),
@@ -776,6 +777,7 @@ public void moveMigratedFiles( File migrationDir, File storeDir, String versionT
             idFilesToDelete = new StoreFile[]{};
             break;
         case Legacy22Store.LEGACY_VERSION:
+        case Legacy23Store.LEGACY_VERSION:
             filesToMove = Collections.emptyList();
             idFilesToDelete = new StoreFile[]{};
             break;
@@ -795,7 +797,10 @@ public void moveMigratedFiles( File migrationDir, File storeDir, String versionT
                 true, // allow to overwrite target files
                 StoreFileType.values() );
 
-        StoreFile.removeTrailers( versionToUpgradeFrom, fileSystem, storeDir, pageCache.pageSize() );
+        if ( !Legacy23Store.LEGACY_VERSION.equals( versionToUpgradeFrom ) )
+        {
+            StoreFile.removeTrailers( versionToUpgradeFrom, fileSystem, storeDir, pageCache.pageSize() );
+        }
 
         File neoStore = new File( storeDir, MetaDataStore.DEFAULT_NAME );
         long logVersion = MetaDataStore.getRecord( pageCache, neoStore, Position.LOG_VERSION );
@@ -807,8 +812,11 @@ public void moveMigratedFiles( File migrationDir, File storeDir, String versionT
         // delete old logs
         legacyLogs.deleteUnusedLogFiles( storeDir );
 
-        // write a check point in the log in order to make recovery work in the newer version
-        new StoreMigratorCheckPointer( storeDir, fileSystem ).checkPoint( logVersion, lastCommittedTx );
+        if ( !Legacy23Store.LEGACY_VERSION.equals( versionToUpgradeFrom ) )
+        {
+            // write a check point in the log in order to make recovery work in the newer version
+            new StoreMigratorCheckPointer( storeDir, fileSystem ).checkPoint( logVersion, lastCommittedTx );
+        }
     }
 
     @Override
@@ -818,6 +826,7 @@ public void rebuildCounts( File storeDir, String versionToMigrateFrom ) throws I
         {
         case Legacy19Store.LEGACY_VERSION:
         case Legacy20Store.LEGACY_VERSION:
+        case Legacy23Store.LEGACY_VERSION:
             // nothing to do
             break;
         case Legacy21Store.LEGACY_VERSION:

community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/StoreUpgrader.java

@@ -333,4 +333,16 @@ public UnexpectedUpgradingStoreVersionException( String filename, String expecte
             super( String.format( MESSAGE, filename, expectedVersion, actualVersion ) );
         }
     }
+
+    public static class DatabaseNotCleanlyShutDown extends UnableToUpgradeException
+    {
+        private static final String MESSAGE =
+                "The database is not cleanly shutdown. The database needs recovery, in order to recover the database, "
+                + "please run the old version of the database on this store.";
+
+        public DatabaseNotCleanlyShutDown()
+        {
+            super( MESSAGE );
+        }
+    }
 }

community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/StoreVersionCheck.java

@@ -59,7 +59,7 @@ record = MetaDataStore.getRecord( pageCache, neostoreFile, STORE_VERSION );
         String storeVersion = MetaDataStore.versionLongToString( record );
         if ( !expectedVersion.equals( storeVersion ) )
         {
-            return new Result( Outcome.unexpectedUpgradingStoreVersion, storeVersion, expectedVersion );
+            return new Result( Outcome.unexpectedUpgradingStoreVersion, storeVersion, neostoreFile.getName() );
         }
 
         return new Result( Outcome.ok, null, neostoreFile.getName() );
@@ -83,7 +83,8 @@ public enum Outcome
             ok( true ),
             missingStoreFile( false ),
             storeVersionNotFound( false ),
-            unexpectedUpgradingStoreVersion( false );
+            unexpectedUpgradingStoreVersion( false ),
+            storeNotCleanlyShutDown( false );
 
             private final boolean success;
 

community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/UpgradableDatabase.java

@@ -20,7 +20,9 @@
 package org.neo4j.kernel.impl.storemigration;
 
 import java.io.File;
+import java.io.IOException;
 
+import org.neo4j.io.fs.FileSystemAbstraction;
 import org.neo4j.kernel.impl.store.CommonAbstractStore;
 import org.neo4j.kernel.impl.store.MetaDataStore;
 import org.neo4j.kernel.impl.storemigration.StoreVersionCheck.Result;
@@ -29,18 +31,26 @@
 import org.neo4j.kernel.impl.storemigration.legacystore.v20.Legacy20Store;
 import org.neo4j.kernel.impl.storemigration.legacystore.v21.Legacy21Store;
 import org.neo4j.kernel.impl.storemigration.legacystore.v22.Legacy22Store;
+import org.neo4j.kernel.impl.storemigration.legacystore.v23.Legacy23Store;
+import org.neo4j.kernel.impl.transaction.log.PhysicalLogFiles;
+import org.neo4j.kernel.impl.transaction.log.entry.VersionAwareLogEntryReader;
+import org.neo4j.kernel.recovery.LatestCheckPointFinder;
+import org.neo4j.kernel.recovery.LatestCheckPointFinder.LatestCheckPoint;
 
 /**
  * Logic to check whether a database version is upgradable to the current version. It looks at the
  * version information found in the store files themselves.
  */
 public class UpgradableDatabase
 {
+    private final FileSystemAbstraction fs;
     private final StoreVersionCheck storeVersionCheck;
     private final LegacyStoreVersionCheck legacyStoreVersionCheck;
 
-    public UpgradableDatabase( StoreVersionCheck storeVersionCheck, LegacyStoreVersionCheck legacyStoreVersionCheck )
+    public UpgradableDatabase( FileSystemAbstraction fs,
+            StoreVersionCheck storeVersionCheck, LegacyStoreVersionCheck legacyStoreVersionCheck )
     {
+        this.fs = fs;
         this.storeVersionCheck = storeVersionCheck;
         this.legacyStoreVersionCheck = legacyStoreVersionCheck;
     }
@@ -84,36 +94,82 @@ public String checkUpgradeable( File storeDirectory )
             return Legacy22Store.LEGACY_VERSION;
         }
 
+        result = checkUpgradeableFor( storeDirectory, Legacy23Store.LEGACY_VERSION );
+        if ( result.outcome.isSuccessful() )
+        {
+            return Legacy23Store.LEGACY_VERSION;
+        }
+
         // report error
-        String path = new File( storeDirectory, result.storeFilename ).getAbsolutePath();
         switch ( result.outcome )
         {
-            case missingStoreFile:
-                throw new StoreUpgrader.UpgradeMissingStoreFilesException( path );
+        case missingStoreFile:
+            throw new StoreUpgrader.UpgradeMissingStoreFilesException( getPathToStoreFile( storeDirectory, result ) );
             case storeVersionNotFound:
-                throw new StoreUpgrader.UpgradingStoreVersionNotFoundException( path );
+                throw new StoreUpgrader.UpgradingStoreVersionNotFoundException(
+                        getPathToStoreFile( storeDirectory, result ) );
             case unexpectedUpgradingStoreVersion:
                 throw new StoreUpgrader.UnexpectedUpgradingStoreVersionException(
-                        path, Legacy21Store.LEGACY_VERSION, result.actualVersion );
+                        getPathToStoreFile( storeDirectory, result ),
+                        Legacy23Store.LEGACY_VERSION, result.actualVersion );
+            case storeNotCleanlyShutDown:
+                throw new StoreUpgrader.DatabaseNotCleanlyShutDown();
             default:
                 throw new IllegalArgumentException( "Unexpected outcome: " + result.outcome.name() );
         }
     }
 
+    private String getPathToStoreFile( File storeDirectory, Result result )
+    {
+        return new File( storeDirectory, result.storeFilename ).getAbsolutePath();
+    }
+
     private Result checkUpgradeableFor( File storeDirectory, String version )
     {
-        Result result = null;
-        for ( StoreFile store : StoreFile.legacyStoreFilesForVersion( version ) )
+        if ( Legacy23Store.LEGACY_VERSION.equals( version ) )
         {
-            String expectedVersion = store.forVersion( version );
-            File storeFile = new File( storeDirectory, store.storeFileName() );
-            result = legacyStoreVersionCheck.hasVersion( storeFile, expectedVersion, store.isOptional() );
+            // check version
+            File neoStore = new File( storeDirectory, MetaDataStore.DEFAULT_NAME );
+            Result result = storeVersionCheck.hasVersion( neoStore, version );
+
             if ( !result.outcome.isSuccessful() )
             {
-                break;
+                return result;
+            }
+
+            PhysicalLogFiles logFiles = new PhysicalLogFiles( storeDirectory, fs );
+            LatestCheckPointFinder latestCheckPointFinder =
+                    new LatestCheckPointFinder( logFiles, fs, new VersionAwareLogEntryReader<>() );
+            try
+            {
+                LatestCheckPoint latestCheckPoint = latestCheckPointFinder.find( logFiles.getHighestLogVersion() );
+                if ( !latestCheckPoint.commitsAfterCheckPoint )
+                {
+                    return new Result( Result.Outcome.ok, null, null );
+                }
+            }
+            catch ( IOException e )
+            {
+                // ignore exception and return db not cleanly shutdown
+            }
+
+            return new Result( Result.Outcome.storeNotCleanlyShutDown, null, null );
+        }
+        else
+        {
+            Result result = null;
+            for ( StoreFile store : StoreFile.legacyStoreFilesForVersion( version ) )
+            {
+                String expectedVersion = store.forVersion( version );
+                File storeFile = new File( storeDirectory, store.storeFileName() );
+                result = legacyStoreVersionCheck.hasVersion( storeFile, expectedVersion, store.isOptional() );
+                if ( !result.outcome.isSuccessful() )
+                {
+                    break;
+                }
             }
+            return result;
         }
-        return result;
     }
 
     public boolean hasCurrentVersion( File storeDir )

community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/legacystore/v22/Legacy22Store.java

@@ -22,8 +22,6 @@
 import java.io.File;
 import java.io.IOException;
 
-import org.neo4j.helpers.UTF8;
-import org.neo4j.kernel.impl.store.CommonAbstractStore;
 import org.neo4j.kernel.impl.storemigration.legacystore.LegacyStore;
 
 /**
@@ -38,31 +36,11 @@ public class Legacy22Store implements LegacyStore
 {
     public static final String LEGACY_VERSION = "v0.A.5";
 
-    private final File storageFileName;
-
-    public Legacy22Store( File storageFileName ) throws IOException
-    {
-        this.storageFileName = storageFileName;
-        assertLegacyAndCurrentVersionHaveSameLength( LEGACY_VERSION, CommonAbstractStore.ALL_STORES_VERSION );
-    }
-
-    /**
-     * Store files that don't need migration are just copied and have their trailing versions replaced
-     * by the current version. For this to work the legacy version and the current version must have the
-     * same encoded length.
-     */
-    static void assertLegacyAndCurrentVersionHaveSameLength( String legacyVersion, String currentVersion )
-    {
-        if ( UTF8.encode( legacyVersion ).length != UTF8.encode( currentVersion ).length )
-        {
-            throw new IllegalStateException( "Encoded version string length must remain the same between versions" );
-        }
-    }
-
     @Override
     public File getStorageFileName()
     {
-        return storageFileName;
+        // not needed
+        throw new UnsupportedOperationException();
     }
 
     @Override