diff --git a/community/dbms/src/main/java/org/neo4j/dbms/ConfigFactory.java b/community/dbms/src/main/java/org/neo4j/dbms/ConfigFactory.java new file mode 100644 index 0000000000000..f24b16d050eb7 --- /dev/null +++ b/community/dbms/src/main/java/org/neo4j/dbms/ConfigFactory.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2002-2016 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package org.neo4j.dbms; + +import java.io.File; +import java.io.IOException; + +import org.neo4j.graphdb.factory.GraphDatabaseSettings; +import org.neo4j.kernel.configuration.Config; + +import static org.neo4j.helpers.collection.MapUtil.load; + +public class ConfigFactory +{ + public static Config readFrom( File file ) + { + try + { + return new Config( load( file ), GraphDatabaseSettings.class, DatabaseManagementSystemSettings.class ); + } + catch ( IOException e ) + { + throw new RuntimeException( String.format( "Could not read configuration file [%s]", + file.getAbsolutePath() ), e ); + } + } +} diff --git a/enterprise/core-edge/pom.xml b/enterprise/core-edge/pom.xml index 11d3192022b83..fd413a82f3006 100644 --- a/enterprise/core-edge/pom.xml +++ b/enterprise/core-edge/pom.xml @@ -57,6 +57,12 @@ + + org.neo4j + neo4j-dbms + ${project.version} + + org.neo4j neo4j-consistency-check diff --git a/enterprise/core-edge/src/main/java/org/neo4j/coreedge/convert/ConvertClassicStoreCommand.java b/enterprise/core-edge/src/main/java/org/neo4j/coreedge/convert/ConvertClassicStoreCommand.java new file mode 100644 index 0000000000000..91e5e01cf79a6 --- /dev/null +++ b/enterprise/core-edge/src/main/java/org/neo4j/coreedge/convert/ConvertClassicStoreCommand.java @@ -0,0 +1,149 @@ +/* + * Copyright (c) 2002-2016 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package org.neo4j.coreedge.convert; + +import java.io.File; +import java.util.Collections; + +import org.neo4j.coreedge.raft.replication.tx.LogIndexTxHeaderEncoding; +import org.neo4j.coreedge.raft.state.DurableStateStorageImporter; +import org.neo4j.coreedge.raft.state.id_allocation.IdAllocationState; +import org.neo4j.graphdb.factory.EnterpriseGraphDatabaseFactory; +import org.neo4j.graphdb.factory.GraphDatabaseFactory; +import org.neo4j.io.fs.DefaultFileSystemAbstraction; +import org.neo4j.kernel.api.exceptions.TransactionFailureException; +import org.neo4j.kernel.impl.api.TransactionCommitProcess; +import org.neo4j.kernel.impl.api.TransactionToApply; +import org.neo4j.kernel.impl.core.DatabasePanicEventGenerator; +import org.neo4j.kernel.impl.store.MetaDataStore; +import org.neo4j.kernel.impl.store.StoreFactory; +import org.neo4j.kernel.impl.store.id.DefaultIdGeneratorFactory; +import org.neo4j.kernel.impl.store.id.IdGenerator; +import org.neo4j.kernel.impl.store.id.IdType; +import org.neo4j.kernel.impl.transaction.log.PhysicalTransactionRepresentation; +import org.neo4j.kernel.impl.transaction.tracing.CommitEvent; +import org.neo4j.kernel.internal.DatabaseHealth; +import org.neo4j.kernel.internal.GraphDatabaseAPI; +import org.neo4j.kernel.internal.KernelEventHandlers; +import org.neo4j.logging.NullLog; +import org.neo4j.logging.NullLogProvider; +import org.neo4j.storageengine.api.TransactionApplicationMode; + +import static org.neo4j.kernel.impl.store.StoreFactory.*; +import static org.neo4j.kernel.impl.store.StoreFactory.PROPERTY_KEY_TOKEN_NAMES_STORE_NAME; +import static org.neo4j.kernel.impl.store.StoreFactory.RELATIONSHIP_TYPE_TOKEN_NAMES_STORE_NAME; +import static org.neo4j.kernel.impl.store.StoreFactory.RELATIONSHIP_TYPE_TOKEN_STORE_NAME; +import static org.neo4j.kernel.impl.store.id.IdType.ARRAY_BLOCK; +import static org.neo4j.kernel.impl.store.id.IdType.LABEL_TOKEN; +import static org.neo4j.kernel.impl.store.id.IdType.LABEL_TOKEN_NAME; +import static org.neo4j.kernel.impl.store.id.IdType.NEOSTORE_BLOCK; +import static org.neo4j.kernel.impl.store.id.IdType.NODE; +import static org.neo4j.kernel.impl.store.id.IdType.NODE_LABELS; +import static org.neo4j.kernel.impl.store.id.IdType.PROPERTY; +import static org.neo4j.kernel.impl.store.id.IdType.PROPERTY_KEY_TOKEN; +import static org.neo4j.kernel.impl.store.id.IdType.PROPERTY_KEY_TOKEN_NAME; +import static org.neo4j.kernel.impl.store.id.IdType.RELATIONSHIP; +import static org.neo4j.kernel.impl.store.id.IdType.RELATIONSHIP_GROUP; +import static org.neo4j.kernel.impl.store.id.IdType.RELATIONSHIP_TYPE_TOKEN; +import static org.neo4j.kernel.impl.store.id.IdType.RELATIONSHIP_TYPE_TOKEN_NAME; +import static org.neo4j.kernel.impl.store.id.IdType.SCHEMA; +import static org.neo4j.kernel.impl.store.id.IdType.STRING_BLOCK; + +public class ConvertClassicStoreCommand +{ + private File databaseDir; + + public ConvertClassicStoreCommand( File databaseDir ) + { + this.databaseDir = databaseDir; + } + + public void execute() throws Throwable + { + appendNullTransactionLogEntry( databaseDir ); + addIdAllocationState( databaseDir ); + } + + private void appendNullTransactionLogEntry( File dbDir) throws TransactionFailureException + { + GraphDatabaseAPI db = (GraphDatabaseAPI) new EnterpriseGraphDatabaseFactory().newEmbeddedDatabase( dbDir ); + + TransactionCommitProcess commitProcess = db.getDependencyResolver().resolveDependency( TransactionCommitProcess.class ); + + PhysicalTransactionRepresentation tx = new PhysicalTransactionRepresentation( Collections.emptyList() ); + byte[] txHeaderBytes = LogIndexTxHeaderEncoding.encodeLogIndexAsTxHeader( -1 ); + tx.setHeader( txHeaderBytes, -1, -1, -1, -1, -1, -1 ); + + commitProcess.commit( new TransactionToApply( tx ), CommitEvent.NULL, TransactionApplicationMode.EXTERNAL ); + + db.shutdown(); + } + + private void addIdAllocationState( File dbDir ) throws Throwable + { + File clusterStateDirectory = new File( dbDir, "cluster-state" ); + + DefaultFileSystemAbstraction fileSystem = new DefaultFileSystemAbstraction(); + DefaultIdGeneratorFactory factory = new DefaultIdGeneratorFactory( fileSystem ); + + long[] highIds = new long[]{ + getHighId( dbDir, factory, NODE, NODE_STORE_NAME ), + getHighId( dbDir, factory, RELATIONSHIP, RELATIONSHIP_STORE_NAME ), + getHighId( dbDir, factory, PROPERTY, PROPERTY_STORE_NAME ), + getHighId( dbDir, factory, STRING_BLOCK, PROPERTY_STRINGS_STORE_NAME ), + getHighId( dbDir, factory, ARRAY_BLOCK, PROPERTY_ARRAYS_STORE_NAME ), + getHighId( dbDir, factory, PROPERTY_KEY_TOKEN, PROPERTY_KEY_TOKEN_STORE_NAME ), + getHighId( dbDir, factory, PROPERTY_KEY_TOKEN_NAME, PROPERTY_KEY_TOKEN_NAMES_STORE_NAME ), + getHighId( dbDir, factory, RELATIONSHIP_TYPE_TOKEN, RELATIONSHIP_TYPE_TOKEN_STORE_NAME ), + getHighId( dbDir, factory, RELATIONSHIP_TYPE_TOKEN_NAME, RELATIONSHIP_TYPE_TOKEN_NAMES_STORE_NAME ), + getHighId( dbDir, factory, LABEL_TOKEN, LABEL_TOKEN_STORE_NAME ), + getHighId( dbDir, factory, LABEL_TOKEN_NAME, LABEL_TOKEN_NAMES_STORE_NAME ), + getHighId( dbDir, factory, NEOSTORE_BLOCK, "" ), + getHighId( dbDir, factory, SCHEMA, SCHEMA_STORE_NAME ), + getHighId( dbDir, factory, NODE_LABELS, NODE_LABELS_STORE_NAME ), + getHighId( dbDir, factory, RELATIONSHIP_GROUP, RELATIONSHIP_GROUP_STORE_NAME )}; + + IdAllocationState state = new IdAllocationState( highIds, -1 ); + + DurableStateStorageImporter storage = new DurableStateStorageImporter<>( + fileSystem, new File( clusterStateDirectory, "id-allocation-state" ), "id-allocation", + new IdAllocationState.Marshal(), + 1000, () -> new DatabaseHealth( + new DatabasePanicEventGenerator( new KernelEventHandlers( NullLog.getInstance() ) ), + NullLog.getInstance() ), NullLogProvider.getInstance() ); + + storage.persist( state ); + + storage.shutdown(); + } + + private long getHighId( File coreDir, DefaultIdGeneratorFactory factory, IdType idType, String store ) + { + IdGenerator idGenerator = factory.open( new File( coreDir, idFile( store ) ), idType.getGrabSize(), idType, -1 ); + long highId = idGenerator.getHighId(); + idGenerator.close(); + return highId; + } + + private String idFile( String store ) + { + return MetaDataStore.DEFAULT_NAME + store + ".id"; + } +} diff --git a/enterprise/core-edge/src/main/java/org/neo4j/coreedge/convert/ConvertNonCoreEdgeStoreCli.java b/enterprise/core-edge/src/main/java/org/neo4j/coreedge/convert/ConvertNonCoreEdgeStoreCli.java new file mode 100644 index 0000000000000..1a26c3db82adf --- /dev/null +++ b/enterprise/core-edge/src/main/java/org/neo4j/coreedge/convert/ConvertNonCoreEdgeStoreCli.java @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2002-2016 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package org.neo4j.coreedge.convert; + +import java.io.File; +import java.io.PrintStream; + +import org.neo4j.dbms.ConfigFactory; +import org.neo4j.dbms.DatabaseManagementSystemSettings; +import org.neo4j.helpers.Args; +import org.neo4j.helpers.ArrayUtil; +import org.neo4j.helpers.Strings; +import org.neo4j.kernel.configuration.Config; +import org.neo4j.kernel.impl.util.Converters; + +import static org.neo4j.helpers.Strings.TAB; +import static org.neo4j.helpers.collection.MapUtil.stringMap; + +public class ConvertNonCoreEdgeStoreCli +{ + public static void main( String[] incomingArguments ) throws Throwable + { + Args args = Args.parse( incomingArguments ); + if ( ArrayUtil.isEmpty( incomingArguments ) ) + { + printUsage( System.out ); + System.exit( 1 ); + } + + String databaseName = args.interpretOption( "database", Converters.mandatory(), s -> s ); + String configPath = args.interpretOption( "config", Converters.mandatory(), s -> s ); + + Config config = ConfigFactory.readFrom( new File( configPath, "neo4j.conf" ) ) + .with( stringMap( DatabaseManagementSystemSettings.active_database.name(), databaseName ) ); + + new ConvertClassicStoreCommand( config.get( DatabaseManagementSystemSettings.database_path ) ).execute(); + } + + private static void printUsage( PrintStream out ) + { + out.println( "Neo4j Classic to Core Format Conversion Tool" ); + for ( String line : Args.splitLongLine( "The classic to core conversion tool is used to convert a classic" + + "Neo4j store into one which has a core friendly format.", 80 ) ) + { + out.println( "\t" + line ); + } + + out.println( "Usage:" ); + out.println("--database "); + out.println("--config "); + } +} diff --git a/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/state/DurableStateStorage.java b/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/state/DurableStateStorage.java index 09e45af839766..6ccb2b8275676 100644 --- a/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/state/DurableStateStorage.java +++ b/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/state/DurableStateStorage.java @@ -110,7 +110,7 @@ public synchronized void persistStoreData( STATE state ) throws IOException } } - private void switchStoreFile() throws IOException + protected void switchStoreFile() throws IOException { currentStoreChannel.close(); diff --git a/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/state/DurableStateStorageImporter.java b/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/state/DurableStateStorageImporter.java new file mode 100644 index 0000000000000..d4040ac829ac9 --- /dev/null +++ b/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/state/DurableStateStorageImporter.java @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2002-2016 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package org.neo4j.coreedge.raft.state; + +import java.io.File; +import java.io.IOException; +import java.util.function.Supplier; + +import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.kernel.impl.transaction.log.PhysicalFlushableChannel; +import org.neo4j.kernel.internal.DatabaseHealth; +import org.neo4j.kernel.lifecycle.LifecycleAdapter; +import org.neo4j.logging.Log; +import org.neo4j.logging.LogProvider; + +public class DurableStateStorageImporter extends DurableStateStorage +{ + public DurableStateStorageImporter( FileSystemAbstraction fileSystemAbstraction, File stateDir, String name, + StateMarshal marshal, int numberOfEntriesBeforeRotation, + Supplier databaseHealthSupplier, LogProvider logProvider ) + throws IOException + { + super( fileSystemAbstraction, stateDir, name, marshal, numberOfEntriesBeforeRotation, databaseHealthSupplier, + logProvider ); + } + + + public void persist( STATE state ) throws IOException + { + super.persistStoreData( state ); + super.switchStoreFile(); + super.persistStoreData( state ); + } +} diff --git a/enterprise/core-edge/src/test/java/org/neo4j/coreedge/raft/state/InMemoryStateStorage.java b/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/state/InMemoryStateStorage.java similarity index 100% rename from enterprise/core-edge/src/test/java/org/neo4j/coreedge/raft/state/InMemoryStateStorage.java rename to enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/state/InMemoryStateStorage.java diff --git a/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/state/id_allocation/IdAllocationState.java b/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/state/id_allocation/IdAllocationState.java index 4573c27888b49..f79247a19bcfd 100644 --- a/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/state/id_allocation/IdAllocationState.java +++ b/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/state/id_allocation/IdAllocationState.java @@ -51,8 +51,9 @@ public IdAllocationState() this( new long[IdType.values().length], -1L ); } - private IdAllocationState( long[] firstUnallocated, + public IdAllocationState( long[] firstUnallocated, long logIndex ) + { this.firstUnallocated = firstUnallocated; this.logIndex = logIndex; diff --git a/enterprise/core-edge/src/main/java/org/neo4j/coreedge/server/core/EnterpriseCoreEditionModule.java b/enterprise/core-edge/src/main/java/org/neo4j/coreedge/server/core/EnterpriseCoreEditionModule.java index 3c9ed20c799d7..b4f34922b2ddc 100644 --- a/enterprise/core-edge/src/main/java/org/neo4j/coreedge/server/core/EnterpriseCoreEditionModule.java +++ b/enterprise/core-edge/src/main/java/org/neo4j/coreedge/server/core/EnterpriseCoreEditionModule.java @@ -107,11 +107,7 @@ import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; import org.neo4j.kernel.DatabaseAvailability; -import org.neo4j.kernel.impl.transaction.log.PhysicalLogFile; -import org.neo4j.kernel.internal.GraphDatabaseAPI; -import org.neo4j.kernel.internal.KernelData; import org.neo4j.kernel.NeoStoreDataSource; -import org.neo4j.kernel.internal.Version; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.api.SchemaWriteGuard; import org.neo4j.kernel.impl.api.TransactionHeaderInformation; @@ -130,8 +126,12 @@ import org.neo4j.kernel.impl.store.stats.IdBasedStoreEntityCounters; import org.neo4j.kernel.impl.transaction.TransactionHeaderInformationFactory; import org.neo4j.kernel.impl.transaction.log.LogicalTransactionStore; +import org.neo4j.kernel.impl.transaction.log.PhysicalLogFile; import org.neo4j.kernel.impl.transaction.log.TransactionIdStore; import org.neo4j.kernel.internal.DatabaseHealth; +import org.neo4j.kernel.internal.GraphDatabaseAPI; +import org.neo4j.kernel.internal.KernelData; +import org.neo4j.kernel.internal.Version; import org.neo4j.kernel.lifecycle.LifeSupport; import org.neo4j.kernel.lifecycle.Lifecycle; import org.neo4j.kernel.lifecycle.LifecycleAdapter; diff --git a/enterprise/core-edge/src/test/java/org/neo4j/coreedge/convert/ConvertNonCoreEdgeStoreCliTest.java b/enterprise/core-edge/src/test/java/org/neo4j/coreedge/convert/ConvertNonCoreEdgeStoreCliTest.java new file mode 100644 index 0000000000000..ddf68357c9b04 --- /dev/null +++ b/enterprise/core-edge/src/test/java/org/neo4j/coreedge/convert/ConvertNonCoreEdgeStoreCliTest.java @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2002-2016 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package org.neo4j.coreedge.convert; + +import org.junit.Test; + +import static junit.framework.TestCase.fail; +import static org.junit.Assert.assertTrue; + +public class ConvertNonCoreEdgeStoreCliTest +{ + @Test + public void shouldIndicateMissingDatabase() throws Throwable + { + try + { + // given + ConvertNonCoreEdgeStoreCli.main( new String[]{""} ); + fail( "Should have thrown IllegalArgumentException" ); + } + catch ( IllegalArgumentException exception ) + { + assertTrue(exception.getMessage(), exception.getMessage().contains( "Missing argument 'database'" ) ); + } + } + + @Test + public void shouldIndicateMissingConfig() throws Throwable + { + try + { + // given + ConvertNonCoreEdgeStoreCli.main( new String[]{"--database", "foo"} ); + fail( "Should have thrown IllegalArgumentException" ); + } + catch ( IllegalArgumentException exception ) + { + assertTrue(exception.getMessage(), exception.getMessage().contains( "Missing argument 'config'" ) ); + } + } + +} \ No newline at end of file diff --git a/enterprise/core-edge/src/test/java/org/neo4j/coreedge/scenarios/ConvertNonCoreEdgeStoreIT.java b/enterprise/core-edge/src/test/java/org/neo4j/coreedge/scenarios/ConvertNonCoreEdgeStoreIT.java new file mode 100644 index 0000000000000..557f3bf0637a4 --- /dev/null +++ b/enterprise/core-edge/src/test/java/org/neo4j/coreedge/scenarios/ConvertNonCoreEdgeStoreIT.java @@ -0,0 +1,137 @@ +/* + * Copyright (c) 2002-2016 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package org.neo4j.coreedge.scenarios; + +import java.io.File; + +import junit.framework.Assert; +import org.junit.After; +import org.junit.Rule; +import org.junit.Test; + +import org.neo4j.coreedge.convert.ConvertClassicStoreCommand; +import org.neo4j.coreedge.discovery.Cluster; +import org.neo4j.coreedge.server.core.CoreGraphDatabase; +import org.neo4j.function.ThrowingSupplier; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.graphdb.Label; +import org.neo4j.graphdb.Node; +import org.neo4j.graphdb.RelationshipType; +import org.neo4j.graphdb.Transaction; +import org.neo4j.graphdb.factory.GraphDatabaseFactory; +import org.neo4j.helpers.collection.IteratorUtil; +import org.neo4j.io.fs.FileUtils; +import org.neo4j.kernel.configuration.Config; +import org.neo4j.test.TargetDirectory; +import org.neo4j.tooling.GlobalGraphOperations; + +import static java.util.concurrent.TimeUnit.SECONDS; + +import static org.hamcrest.Matchers.greaterThan; + +import static org.neo4j.coreedge.server.CoreEdgeClusterSettings.raft_advertised_address; +import static org.neo4j.graphdb.Label.label; +import static org.neo4j.helpers.collection.Iterables.count; +import static org.neo4j.test.Assert.assertEventually; + +public class ConvertNonCoreEdgeStoreIT +{ + @Rule + public final TargetDirectory.TestDirectory dir = TargetDirectory.testDirForTest( getClass() ); + + private Cluster cluster; + + @After + public void shutdown() + { + if ( cluster != null ) + { + cluster.shutdown(); + } + } + + @Test + public void shouldReplicateTransactionToCoreServers() throws Throwable + { + // given + File dbDir = dir.directory(); + FileUtils.deleteRecursively( dbDir ); + + File classicNeo4jStore = createClassicNeo4jStore( dbDir, 2000 ); + + for ( int serverId = 0; serverId < 3; serverId++ ) + { + File destination = new File( dbDir, "server-core-" + serverId ); + FileUtils.copyRecursively( classicNeo4jStore, destination ); + new ConvertClassicStoreCommand( destination ).execute(); + } + + cluster = Cluster.start( dbDir, 3, 0 ); + + // when + GraphDatabaseService coreDB = cluster.findLeader( 5000 ); + + try ( Transaction tx = coreDB.beginTx() ) + { + Node node = coreDB.createNode( label( "boo" ) ); + node.setProperty( "foobar", "baz_bat" ); + tx.success(); + } + + // then + for ( final CoreGraphDatabase db : cluster.coreServers() ) + { + try ( Transaction tx = db.beginTx() ) + { + ThrowingSupplier nodeCount = () -> count( db.getAllNodes() ); + + Config config = db.getDependencyResolver().resolveDependency( Config.class ); + + assertEventually( "node to appear on core server " + config.get( raft_advertised_address ), nodeCount, + greaterThan( 0L ), 15, SECONDS ); + + Assert.assertEquals( 2001, IteratorUtil.count( GlobalGraphOperations.at( db ).getAllNodes() ) ); + + tx.success(); + } + } + } + + private File createClassicNeo4jStore( File base, int nodesToCreate ) + { + File existingDbDir = new File( base, "existing" ); + GraphDatabaseService db = new GraphDatabaseFactory().newEmbeddedDatabase( existingDbDir ); + + for ( int i = 0; i < (nodesToCreate / 2); i++ ) + { + try ( Transaction tx = db.beginTx() ) + { + Node node1 = db.createNode( Label.label( "Label-" + i ) ); + Node node2 = db.createNode( Label.label( "Label-" + i ) ); + node1.createRelationshipTo( node2, RelationshipType.withName( "REL-" + i ) ); + tx.success(); + } + } + + db.shutdown(); + + return existingDbDir; + } +} diff --git a/packaging/standalone/src/main/distribution/shell-scripts/bin/neo4j-admin b/packaging/standalone/src/main/distribution/shell-scripts/bin/neo4j-admin index 6fc4e3cb6e879..aea2d3a4abe1b 100755 --- a/packaging/standalone/src/main/distribution/shell-scripts/bin/neo4j-admin +++ b/packaging/standalone/src/main/distribution/shell-scripts/bin/neo4j-admin @@ -93,6 +93,136 @@ cmd_import() { esac } +find_java_home() { + [[ "${JAVA_HOME:-}" ]] && return + + case "${DIST_OS}" in + "macosx") + JAVA_HOME="$(/usr/libexec/java_home -v 1.8)" + ;; + "gentoo") + JAVA_HOME="$(java-config --jre-home)" + ;; + esac +} + +find_java_cmd() { + [[ "${JAVA_CMD:-}" ]] && return + detect_os + find_java_home + + if [[ "${JAVA_HOME:-}" ]] ; then + JAVA_CMD="${JAVA_HOME}/bin/java" + else + if [ "${DIST_OS}" != "macosx" ] ; then + # Don't use default java on Darwin because it displays a misleading dialog box + JAVA_CMD="$(which java || true)" + fi + fi + + if [[ ! "${JAVA_CMD:-}" ]]; then + echo "ERROR: Unable to find Java executable." + show_java_help + exit 1 + fi +} + +detect_os() { + if uname -s | grep -q Darwin; then + DIST_OS="macosx" + elif [[ -e /etc/gentoo-release ]]; then + DIST_OS="gentoo" + else + DIST_OS="other" + fi +} + +setup_neo4jhome_and_script() { + SCRIPT=$0 + + cd "$(dirname "${SCRIPT}")" + SCRIPT="$(basename "${SCRIPT}")" + + while [ -L "${SCRIPT}" ] + do + SCRIPT="$(readlink "${SCRIPT}")" + cd "$(dirname "${SCRIPT}")" + SCRIPT="$(basename "${SCRIPT}")" + done + + NEO4J_HOME="$(cd "$(dirname "${SCRIPT}")/.." && dirs -l +0)" +} + +build_classpath() { + CLASSPATH="${NEO4J_HOME}/lib/*:${NEO4J_HOME}/plugins/*" +} + +setup_paths() { + [[ "${NEO4J_CONFIG:-}" ]] || NEO4J_CONFIG="${NEO4J_HOME}/conf" +} + +check_java() { + find_java_cmd + + version_command="${JAVA_CMD} -version" + + JAVA_VERSION=$(${version_command} 2>&1 | awk -F '"' '/version/ {print $2}') + if [[ "${JAVA_VERSION}" < "1.8" ]]; then + echo "ERROR! Neo4j cannot be started using java version ${JAVA_VERSION}. " + show_java_help + exit 1 + fi + + if ! (${version_command} 2>&1 | egrep -q "(Java HotSpot\\(TM\\)|OpenJDK) (64-Bit Server|Server|Client) VM"); then + echo "WARNING! You are using an unsupported Java runtime. " + show_java_help + fi +} + +cmd_core_convert_args() { + declare database='' + + while [[ -n "${1:-}" ]]; do + declare arg="$1" + shift + case "${arg}" in + -h|--help) + usage + ;; + -d) + if [[ -n "${1:-}" ]]; then + database="${1}" + shift + fi + ;; + --database=*) + set -- "-d" "${arg#*=}" "$@" + ;; + --database) + ;; + *) + bad_usage "unrecognized core-convert argument '${arg}'" + ;; + esac + done + + [[ -z "${database}" ]] && bad_usage "you must provide the --database option with an argument" + + retval=${database} +} + +cmd_core_convert() { + cmd_core_convert_args "$@" + declare -r database="${retval}" + + setup_neo4jhome_and_script + cd "${NEO4J_HOME}" + setup_paths + check_java + build_classpath + exec "${JAVA_CMD}" -cp "${CLASSPATH}" -Dfile.encoding=UTF-8 "org.neo4j.coreedge.convert.ConvertNonCoreEdgeStoreCli" --database=${database} --config=${NEO4J_CONFIG} +} + bad_usage() { echo "Error: $1" >&2 echo >&2 @@ -112,6 +242,10 @@ usage() { Import a database from a pre-3.0 Neo4j installation. is the database location (e.g. /data/graph.db). + ${PROGRAM} core-convert --database= + + Converts a database created in a non core-edge Neo4j installation into one that is core format friendly. + ${PROGRAM} help Display this help text. @@ -125,6 +259,7 @@ main() { shift case "${command}" in import) cmd_import "$@" ;; + core-convert) cmd_core_convert "$@" ;; help|--help|-h) usage ;; *) bad_usage "unrecognised command '${command}'" ;; esac diff --git a/packaging/standalone/src/tests/shell-scripts/test-admin-args.sh b/packaging/standalone/src/tests/shell-scripts/test-admin-args.sh index cd87160bcb9d9..34c34a2c12a57 100755 --- a/packaging/standalone/src/tests/shell-scripts/test-admin-args.sh +++ b/packaging/standalone/src/tests/shell-scripts/test-admin-args.sh @@ -21,7 +21,9 @@ test_expect_success "should fail if command is unrecognized" " test_expect_success "should fail if --database is missing" " assert_failure_with_stderr 'you must provide the --database option with an argument' \ - neo4j-home/bin/neo4j-admin import --from=/foo --mode=database + neo4j-home/bin/neo4j-admin import --from=/foo --mode=database && + assert_failure_with_stderr 'you must provide the --database option with an argument' \ + neo4j-home/bin/neo4j-admin core-convert " test_expect_success "should fail if --from is missing" " diff --git a/packaging/standalone/src/tests/shell-scripts/test-convert-classic.sh b/packaging/standalone/src/tests/shell-scripts/test-convert-classic.sh new file mode 100755 index 0000000000000..7b59d54aa7c43 --- /dev/null +++ b/packaging/standalone/src/tests/shell-scripts/test-convert-classic.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +test_description="Test neo4j-admin core-convert" + +. ./lib/sharness.sh +fake_install + +test_expect_success "should invoke convert classic store main class, pass in database name and config directory" " + neo4j-home/bin/neo4j-admin core-convert --database=foo.db && + test_expect_java_arg 'org.neo4j.coreedge.convert.ConvertNonCoreEdgeStoreCli' && + test_expect_java_arg '--database=foo.db' && + test_expect_java_arg '--config=$(neo4j_home)/conf' +" + +test_done \ No newline at end of file