Tool for converting Classic neo4j store to core format

* Adds 'core-convert' command into neo4j-admin tool
* The tool adds a dummy transaction which is used to set
  a RAFT index of -1 in the header of the last transaction
  the store as seen which allows us to start up correctly.
* Also hydrates the database with id-allocation state needed
  to startup the ID state machine correctly.

community/dbms/src/main/java/org/neo4j/dbms/ConfigFactory.java

@@ -0,0 +1,44 @@
+/*
+ * Copyright (c) 2002-2016 "Neo Technology,"
+ * Network Engine for Objects in Lund AB [http://neotechnology.com]
+ *
+ * This file is part of Neo4j.
+ *
+ * Neo4j is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.neo4j.dbms;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.neo4j.graphdb.factory.GraphDatabaseSettings;
+import org.neo4j.kernel.configuration.Config;
+
+import static org.neo4j.helpers.collection.MapUtil.load;
+
+public class ConfigFactory
+{
+    public static Config readFrom( File file )
+    {
+        try
+        {
+            return new Config( load( file ), GraphDatabaseSettings.class, DatabaseManagementSystemSettings.class );
+        }
+        catch ( IOException e )
+        {
+            throw new RuntimeException( String.format( "Could not read configuration file [%s]",
+                    file.getAbsolutePath() ), e );
+        }
+    }
+}

enterprise/core-edge/pom.xml

@@ -57,6 +57,12 @@
           </exclusions>
         </dependency>
 
+      <dependency>
+        <groupId>org.neo4j</groupId>
+        <artifactId>neo4j-dbms</artifactId>
+        <version>${project.version}</version>
+      </dependency>
+
       <dependency>
         <groupId>org.neo4j</groupId>
         <artifactId>neo4j-consistency-check</artifactId>

enterprise/core-edge/src/main/java/org/neo4j/coreedge/convert/ConvertClassicStoreCommand.java

@@ -0,0 +1,149 @@
+/*
+ * Copyright (c) 2002-2016 "Neo Technology,"
+ * Network Engine for Objects in Lund AB [http://neotechnology.com]
+ *
+ * This file is part of Neo4j.
+ *
+ * Neo4j is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.neo4j.coreedge.convert;
+
+import java.io.File;
+import java.util.Collections;
+
+import org.neo4j.coreedge.raft.replication.tx.LogIndexTxHeaderEncoding;
+import org.neo4j.coreedge.raft.state.DurableStateStorageImporter;
+import org.neo4j.coreedge.raft.state.id_allocation.IdAllocationState;
+import org.neo4j.graphdb.factory.EnterpriseGraphDatabaseFactory;
+import org.neo4j.graphdb.factory.GraphDatabaseFactory;
+import org.neo4j.io.fs.DefaultFileSystemAbstraction;
+import org.neo4j.kernel.api.exceptions.TransactionFailureException;
+import org.neo4j.kernel.impl.api.TransactionCommitProcess;
+import org.neo4j.kernel.impl.api.TransactionToApply;
+import org.neo4j.kernel.impl.core.DatabasePanicEventGenerator;
+import org.neo4j.kernel.impl.store.MetaDataStore;
+import org.neo4j.kernel.impl.store.StoreFactory;
+import org.neo4j.kernel.impl.store.id.DefaultIdGeneratorFactory;
+import org.neo4j.kernel.impl.store.id.IdGenerator;
+import org.neo4j.kernel.impl.store.id.IdType;
+import org.neo4j.kernel.impl.transaction.log.PhysicalTransactionRepresentation;
+import org.neo4j.kernel.impl.transaction.tracing.CommitEvent;
+import org.neo4j.kernel.internal.DatabaseHealth;
+import org.neo4j.kernel.internal.GraphDatabaseAPI;
+import org.neo4j.kernel.internal.KernelEventHandlers;
+import org.neo4j.logging.NullLog;
+import org.neo4j.logging.NullLogProvider;
+import org.neo4j.storageengine.api.TransactionApplicationMode;
+
+import static org.neo4j.kernel.impl.store.StoreFactory.*;
+import static org.neo4j.kernel.impl.store.StoreFactory.PROPERTY_KEY_TOKEN_NAMES_STORE_NAME;
+import static org.neo4j.kernel.impl.store.StoreFactory.RELATIONSHIP_TYPE_TOKEN_NAMES_STORE_NAME;
+import static org.neo4j.kernel.impl.store.StoreFactory.RELATIONSHIP_TYPE_TOKEN_STORE_NAME;
+import static org.neo4j.kernel.impl.store.id.IdType.ARRAY_BLOCK;
+import static org.neo4j.kernel.impl.store.id.IdType.LABEL_TOKEN;
+import static org.neo4j.kernel.impl.store.id.IdType.LABEL_TOKEN_NAME;
+import static org.neo4j.kernel.impl.store.id.IdType.NEOSTORE_BLOCK;
+import static org.neo4j.kernel.impl.store.id.IdType.NODE;
+import static org.neo4j.kernel.impl.store.id.IdType.NODE_LABELS;
+import static org.neo4j.kernel.impl.store.id.IdType.PROPERTY;
+import static org.neo4j.kernel.impl.store.id.IdType.PROPERTY_KEY_TOKEN;
+import static org.neo4j.kernel.impl.store.id.IdType.PROPERTY_KEY_TOKEN_NAME;
+import static org.neo4j.kernel.impl.store.id.IdType.RELATIONSHIP;
+import static org.neo4j.kernel.impl.store.id.IdType.RELATIONSHIP_GROUP;
+import static org.neo4j.kernel.impl.store.id.IdType.RELATIONSHIP_TYPE_TOKEN;
+import static org.neo4j.kernel.impl.store.id.IdType.RELATIONSHIP_TYPE_TOKEN_NAME;
+import static org.neo4j.kernel.impl.store.id.IdType.SCHEMA;
+import static org.neo4j.kernel.impl.store.id.IdType.STRING_BLOCK;
+
+public class ConvertClassicStoreCommand
+{
+    private File databaseDir;
+
+    public ConvertClassicStoreCommand( File databaseDir )
+    {
+        this.databaseDir = databaseDir;
+    }
+
+    public void execute() throws Throwable
+    {
+        appendNullTransactionLogEntry( databaseDir );
+        addIdAllocationState( databaseDir );
+    }
+
+    private void appendNullTransactionLogEntry( File dbDir) throws TransactionFailureException
+    {
+        GraphDatabaseAPI db = (GraphDatabaseAPI) new EnterpriseGraphDatabaseFactory().newEmbeddedDatabase( dbDir );
+
+        TransactionCommitProcess commitProcess = db.getDependencyResolver().resolveDependency( TransactionCommitProcess.class );
+
+        PhysicalTransactionRepresentation tx = new PhysicalTransactionRepresentation( Collections.emptyList() );
+        byte[] txHeaderBytes = LogIndexTxHeaderEncoding.encodeLogIndexAsTxHeader( -1 );
+        tx.setHeader( txHeaderBytes, -1, -1, -1, -1, -1, -1 );
+
+        commitProcess.commit( new TransactionToApply( tx ), CommitEvent.NULL, TransactionApplicationMode.EXTERNAL );
+
+        db.shutdown();
+    }
+
+    private void addIdAllocationState( File dbDir ) throws Throwable
+    {
+        File clusterStateDirectory = new File( dbDir, "cluster-state" );
+
+        DefaultFileSystemAbstraction fileSystem = new DefaultFileSystemAbstraction();
+        DefaultIdGeneratorFactory factory = new DefaultIdGeneratorFactory( fileSystem );
+
+        long[] highIds = new long[]{
+                getHighId( dbDir, factory, NODE, NODE_STORE_NAME ),
+                getHighId( dbDir, factory, RELATIONSHIP, RELATIONSHIP_STORE_NAME ),
+                getHighId( dbDir, factory, PROPERTY, PROPERTY_STORE_NAME ),
+                getHighId( dbDir, factory, STRING_BLOCK, PROPERTY_STRINGS_STORE_NAME ),
+                getHighId( dbDir, factory, ARRAY_BLOCK, PROPERTY_ARRAYS_STORE_NAME ),
+                getHighId( dbDir, factory, PROPERTY_KEY_TOKEN, PROPERTY_KEY_TOKEN_STORE_NAME ),
+                getHighId( dbDir, factory, PROPERTY_KEY_TOKEN_NAME, PROPERTY_KEY_TOKEN_NAMES_STORE_NAME ),
+                getHighId( dbDir, factory, RELATIONSHIP_TYPE_TOKEN, RELATIONSHIP_TYPE_TOKEN_STORE_NAME ),
+                getHighId( dbDir, factory, RELATIONSHIP_TYPE_TOKEN_NAME, RELATIONSHIP_TYPE_TOKEN_NAMES_STORE_NAME ),
+                getHighId( dbDir, factory, LABEL_TOKEN, LABEL_TOKEN_STORE_NAME ),
+                getHighId( dbDir, factory, LABEL_TOKEN_NAME, LABEL_TOKEN_NAMES_STORE_NAME ),
+                getHighId( dbDir, factory, NEOSTORE_BLOCK, "" ),
+                getHighId( dbDir, factory, SCHEMA, SCHEMA_STORE_NAME ),
+                getHighId( dbDir, factory, NODE_LABELS, NODE_LABELS_STORE_NAME ),
+                getHighId( dbDir, factory, RELATIONSHIP_GROUP, RELATIONSHIP_GROUP_STORE_NAME )};
+
+        IdAllocationState state = new IdAllocationState( highIds, -1 );
+
+        DurableStateStorageImporter<IdAllocationState> storage = new DurableStateStorageImporter<>(
+                fileSystem, new File( clusterStateDirectory, "id-allocation-state" ), "id-allocation",
+                new IdAllocationState.Marshal(),
+                1000, () -> new DatabaseHealth(
+                new DatabasePanicEventGenerator( new KernelEventHandlers( NullLog.getInstance() ) ),
+                NullLog.getInstance() ), NullLogProvider.getInstance() );
+
+        storage.persist( state );
+
+        storage.shutdown();
+    }
+
+    private long getHighId( File coreDir, DefaultIdGeneratorFactory factory, IdType idType, String store )
+    {
+        IdGenerator idGenerator = factory.open( new File( coreDir, idFile( store ) ), idType.getGrabSize(), idType, -1 );
+        long highId = idGenerator.getHighId();
+        idGenerator.close();
+        return highId;
+    }
+
+    private String idFile( String store )
+    {
+        return MetaDataStore.DEFAULT_NAME + store + ".id";
+    }
+}

enterprise/core-edge/src/main/java/org/neo4j/coreedge/convert/ConvertNonCoreEdgeStoreCli.java

@@ -0,0 +1,69 @@
+/*
+ * Copyright (c) 2002-2016 "Neo Technology,"
+ * Network Engine for Objects in Lund AB [http://neotechnology.com]
+ *
+ * This file is part of Neo4j.
+ *
+ * Neo4j is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.neo4j.coreedge.convert;
+
+import java.io.File;
+import java.io.PrintStream;
+
+import org.neo4j.dbms.ConfigFactory;
+import org.neo4j.dbms.DatabaseManagementSystemSettings;
+import org.neo4j.helpers.Args;
+import org.neo4j.helpers.ArrayUtil;
+import org.neo4j.helpers.Strings;
+import org.neo4j.kernel.configuration.Config;
+import org.neo4j.kernel.impl.util.Converters;
+
+import static org.neo4j.helpers.Strings.TAB;
+import static org.neo4j.helpers.collection.MapUtil.stringMap;
+
+public class ConvertNonCoreEdgeStoreCli
+{
+    public static void main( String[] incomingArguments ) throws Throwable
+    {
+        Args args = Args.parse( incomingArguments );
+        if ( ArrayUtil.isEmpty( incomingArguments ) )
+        {
+            printUsage( System.out );
+            System.exit( 1 );
+        }
+
+        String databaseName = args.interpretOption( "database", Converters.<String>mandatory(), s -> s );
+        String configPath = args.interpretOption( "config", Converters.<String>mandatory(), s -> s );
+
+        Config config = ConfigFactory.readFrom( new File( configPath, "neo4j.conf" ) )
+                .with( stringMap( DatabaseManagementSystemSettings.active_database.name(), databaseName ) );
+
+        new ConvertClassicStoreCommand( config.get( DatabaseManagementSystemSettings.database_path ) ).execute();
+    }
+
+    private static void printUsage( PrintStream out )
+    {
+        out.println( "Neo4j Classic to Core Format Conversion Tool" );
+        for ( String line : Args.splitLongLine( "The classic  to core conversion tool is used to convert a classic"
+                + "Neo4j store into one which has a core friendly format.", 80 ) )
+        {
+            out.println( "\t" + line );
+        }
+
+        out.println( "Usage:" );
+        out.println("--database <database-name>");
+        out.println("--config <path-to-config-directory>");
+    }
+}

enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/state/DurableStateStorage.java

@@ -110,7 +110,7 @@ public synchronized void persistStoreData( STATE state ) throws IOException
         }
     }
 
-    private void switchStoreFile() throws IOException
+    protected void switchStoreFile() throws IOException
     {
         currentStoreChannel.close();
 

enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/state/DurableStateStorageImporter.java

@@ -0,0 +1,51 @@
+/*
+ * Copyright (c) 2002-2016 "Neo Technology,"
+ * Network Engine for Objects in Lund AB [http://neotechnology.com]
+ *
+ * This file is part of Neo4j.
+ *
+ * Neo4j is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.neo4j.coreedge.raft.state;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.function.Supplier;
+
+import org.neo4j.io.fs.FileSystemAbstraction;
+import org.neo4j.kernel.impl.transaction.log.PhysicalFlushableChannel;
+import org.neo4j.kernel.internal.DatabaseHealth;
+import org.neo4j.kernel.lifecycle.LifecycleAdapter;
+import org.neo4j.logging.Log;
+import org.neo4j.logging.LogProvider;
+
+public class DurableStateStorageImporter<STATE> extends DurableStateStorage<STATE>
+{
+    public DurableStateStorageImporter( FileSystemAbstraction fileSystemAbstraction, File stateDir, String name,
+                                        StateMarshal<STATE> marshal, int numberOfEntriesBeforeRotation,
+                                        Supplier<DatabaseHealth> databaseHealthSupplier, LogProvider logProvider )
+            throws IOException
+    {
+        super( fileSystemAbstraction, stateDir, name, marshal, numberOfEntriesBeforeRotation, databaseHealthSupplier,
+                logProvider );
+    }
+
+
+    public void persist( STATE state ) throws IOException
+    {
+        super.persistStoreData( state );
+        super.switchStoreFile();
+        super.persistStoreData( state );
+    }
+}

enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/state/id_allocation/IdAllocationState.java

@@ -51,8 +51,9 @@ public IdAllocationState()
         this( new long[IdType.values().length], -1L );
     }
 
-    private IdAllocationState( long[] firstUnallocated,
+    public IdAllocationState( long[] firstUnallocated,
                                long logIndex )
+
     {
         this.firstUnallocated = firstUnallocated;
         this.logIndex = logIndex;

enterprise/core-edge/src/main/java/org/neo4j/coreedge/server/core/EnterpriseCoreEditionModule.java

@@ -107,11 +107,7 @@
 import org.neo4j.io.fs.FileSystemAbstraction;
 import org.neo4j.io.pagecache.PageCache;
 import org.neo4j.kernel.DatabaseAvailability;
-import org.neo4j.kernel.impl.transaction.log.PhysicalLogFile;
-import org.neo4j.kernel.internal.GraphDatabaseAPI;
-import org.neo4j.kernel.internal.KernelData;
 import org.neo4j.kernel.NeoStoreDataSource;
-import org.neo4j.kernel.internal.Version;
 import org.neo4j.kernel.configuration.Config;
 import org.neo4j.kernel.impl.api.SchemaWriteGuard;
 import org.neo4j.kernel.impl.api.TransactionHeaderInformation;
@@ -130,8 +126,12 @@
 import org.neo4j.kernel.impl.store.stats.IdBasedStoreEntityCounters;
 import org.neo4j.kernel.impl.transaction.TransactionHeaderInformationFactory;
 import org.neo4j.kernel.impl.transaction.log.LogicalTransactionStore;
+import org.neo4j.kernel.impl.transaction.log.PhysicalLogFile;
 import org.neo4j.kernel.impl.transaction.log.TransactionIdStore;
 import org.neo4j.kernel.internal.DatabaseHealth;
+import org.neo4j.kernel.internal.GraphDatabaseAPI;
+import org.neo4j.kernel.internal.KernelData;
+import org.neo4j.kernel.internal.Version;
 import org.neo4j.kernel.lifecycle.LifeSupport;
 import org.neo4j.kernel.lifecycle.Lifecycle;
 import org.neo4j.kernel.lifecycle.LifecycleAdapter;