Skip to content

Commit

Permalink
Allow format selection based on available store without necessity
Browse files Browse the repository at this point in the history 
to perform upgrade to latest format always.

To allow users to use already available databases without necessity
of upgrade we will allow now automatic database format selection
during startup.
By introducing this feature we will allow users that can't perform
store format upgrade still to use new features and fixes that does not require
new store format.
  • Loading branch information
@MishaDemianenko
MishaDemianenko committed Nov 27, 2017
1 parent 179456a commit 3d31f75
Show file tree
Hide file tree
Showing 6 changed files with 137 additions and 44 deletions.
22 changes: 22 additions & 0 deletions community/kernel/src/main/java/org/neo4j/kernel/impl/store/format/RecordFormatSelector.java
View file
Expand Up @@ -219,6 +219,28 @@ public static RecordFormats selectForStoreOrConfig( Config config, File storeDir
return DEFAULT_FORMAT;
}

/**
* Check if store and configured formats are compatible. In case if format is not configured or store does not
* exist yet - we consider formats as compatible.
* @param config configuration parameters
* @param storeDir directory with the store
* @param fs the file system
* @param pageCache page cache to read store files
* @param logProvider log provider
* @return true if configured and actual format is compatible, false otherwise.
*/
public static boolean isStoreAndConfigFormatsCompatible( Config config, File storeDir, FileSystemAbstraction fs,
PageCache pageCache, LogProvider logProvider )
{
RecordFormats configuredFormat = loadRecordFormat( configuredRecordFormat( config ) );

RecordFormats currentFormat = selectForStore( storeDir, fs, pageCache, logProvider );

return (configuredFormat == null) || (currentFormat == null) ||
(currentFormat.getFormatFamily().equals( configuredFormat.getFormatFamily() ) &&
(currentFormat.generation() == configuredFormat.generation()));
}

/**
* Select explicitly configured record format (via given {@code config}) or format from the store. If store does
* not exist or has old format ({@link RecordFormats#generation()}) than this method returns
Expand Down
36 changes: 23 additions & 13 deletions community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/StoreUpgrader.java
View file
Expand Up @@ -33,6 +33,7 @@
import org.neo4j.io.fs.FileSystemAbstraction;
import org.neo4j.io.pagecache.PageCache;
import org.neo4j.kernel.configuration.Config;
import org.neo4j.kernel.impl.store.format.RecordFormatSelector;
import org.neo4j.kernel.impl.storemigration.monitoring.MigrationProgressMonitor;
import org.neo4j.kernel.impl.util.monitoring.ProgressReporter;
import org.neo4j.kernel.internal.Version;
Expand Down Expand Up @@ -75,6 +76,7 @@ public class StoreUpgrader
private final FileSystemAbstraction fileSystem;
private final PageCache pageCache;
private final Log log;
private final LogProvider logProvider;

public StoreUpgrader( UpgradableDatabase upgradableDatabase, MigrationProgressMonitor progressMonitor, Config
config, FileSystemAbstraction fileSystem, PageCache pageCache, LogProvider logProvider )
Expand All @@ -84,6 +86,7 @@ public StoreUpgrader( UpgradableDatabase upgradableDatabase, MigrationProgressMo
this.fileSystem = fileSystem;
this.config = config;
this.pageCache = pageCache;
this.logProvider = logProvider;
this.log = logProvider.getLog( getClass() );
}

Expand Down Expand Up @@ -115,11 +118,18 @@ public void migrateIfNeeded( File storeDirectory )
return;
}

if ( !isUpgradeAllowed() )
if ( isUpgradeAllowed() )
{
migrateStore( storeDirectory, migrationDirectory, migrationStateFile );
}
else if ( !RecordFormatSelector.isStoreAndConfigFormatsCompatible( config, storeDirectory, fileSystem, pageCache, logProvider ) )
{
throw new UpgradeNotAllowedByConfigurationException();
}
}

private void migrateStore( File storeDirectory, File migrationDirectory, File migrationStateFile )
{
// One or more participants would like to do migration
progressMonitor.started( participants.size() );

Expand Down Expand Up @@ -273,50 +283,50 @@ public UnableToUpgradeException( String message, Throwable cause )
super( message, cause );
}

public UnableToUpgradeException( String message )
UnableToUpgradeException( String message )
{
super( message );
}
}

public static class UpgradeMissingStoreFilesException extends UnableToUpgradeException
static class UpgradeMissingStoreFilesException extends UnableToUpgradeException
{
private static final String MESSAGE = "Missing required store file '%s'.";

public UpgradeMissingStoreFilesException( String filenameExpectedToExist )
UpgradeMissingStoreFilesException( String filenameExpectedToExist )
{
super( String.format( MESSAGE, filenameExpectedToExist ) );
}
}

public static class UpgradingStoreVersionNotFoundException extends UnableToUpgradeException
static class UpgradingStoreVersionNotFoundException extends UnableToUpgradeException
{
private static final String MESSAGE =
"'%s' does not contain a store version, please ensure that the original database was shut down in a " +
"clean state.";

public UpgradingStoreVersionNotFoundException( String filenameWithoutStoreVersion )
UpgradingStoreVersionNotFoundException( String filenameWithoutStoreVersion )
{
super( String.format( MESSAGE, filenameWithoutStoreVersion ) );
}
}

public static class UnexpectedUpgradingStoreVersionException extends UnableToUpgradeException
{
protected static final String MESSAGE =
static final String MESSAGE =
"Not possible to upgrade a store with version '%s' to current store version `%s` (Neo4j %s).";

public UnexpectedUpgradingStoreVersionException( String fileVersion, String currentVersion )
UnexpectedUpgradingStoreVersionException( String fileVersion, String currentVersion )
{
super( String.format( MESSAGE, fileVersion, currentVersion, Version.getNeo4jVersion() ) );
}
}

public static class AttemptedDowngradeException extends UnableToUpgradeException
{
protected static final String MESSAGE = "Downgrading stores are not supported.";
static final String MESSAGE = "Downgrading stores are not supported.";

public AttemptedDowngradeException()
AttemptedDowngradeException()
{
super( MESSAGE );
}
Expand All @@ -327,19 +337,19 @@ public static class UnexpectedUpgradingStoreFormatException extends UnableToUpgr
protected static final String MESSAGE =
"This is an enterprise-only store. Please configure '%s' to open.";

public UnexpectedUpgradingStoreFormatException()
UnexpectedUpgradingStoreFormatException()
{
super( String.format( MESSAGE, GraphDatabaseSettings.record_format.name() ) );
}
}

public static class DatabaseNotCleanlyShutDownException extends UnableToUpgradeException
static class DatabaseNotCleanlyShutDownException extends UnableToUpgradeException
{
private static final String MESSAGE =
"The database is not cleanly shutdown. The database needs recovery, in order to recover the database, "
+ "please run the old version of the database on this store.";

public DatabaseNotCleanlyShutDownException()
DatabaseNotCleanlyShutDownException()
{
super( MESSAGE );
}
Expand Down
9 changes: 2 additions & 7 deletions community/neo4j/src/test/java/db/DatabaseStartupTest.java
View file
Expand Up @@ -33,15 +33,13 @@
import org.neo4j.io.pagecache.impl.muninn.StandalonePageCacheFactory;
import org.neo4j.kernel.impl.store.MetaDataStore;
import org.neo4j.kernel.impl.storemigration.StoreUpgrader;
import org.neo4j.kernel.impl.storemigration.UpgradeNotAllowedByConfigurationException;
import org.neo4j.kernel.lifecycle.LifecycleException;
import org.neo4j.test.TestGraphDatabaseFactory;
import org.neo4j.test.rule.TestDirectory;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.neo4j.graphdb.factory.GraphDatabaseSettings.allow_upgrade;

public class DatabaseStartupTest
{
Expand Down Expand Up @@ -80,11 +78,8 @@ public void startTheDatabaseWithWrongVersionShouldFailWithUpgradeNotAllowed() th
{
// then
assertTrue( ex.getCause() instanceof LifecycleException );
assertTrue( ex.getCause().getCause() instanceof UpgradeNotAllowedByConfigurationException );
assertEquals( "Neo4j cannot be started because the database files require upgrading and upgrades are " +
"disabled in the configuration. Please set '" + allow_upgrade.name() + "' to 'true' in your " +
"configuration file and try again.",
ex.getCause().getCause().getMessage());
assertTrue( ex.getCause().getCause() instanceof IllegalArgumentException );
assertEquals( "Unknown store version 'bad'", ex.getCause().getCause().getMessage() );
}
}

Expand Down
93 changes: 82 additions & 11 deletions community/neo4j/src/test/java/migration/RecordFormatMigrationIT.java
View file
Expand Up @@ -19,6 +19,7 @@
*/
package migration;

import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;

Expand All @@ -29,25 +30,81 @@
import org.neo4j.graphdb.Transaction;
import org.neo4j.graphdb.factory.GraphDatabaseFactory;
import org.neo4j.helpers.Exceptions;
import org.neo4j.kernel.configuration.Settings;
import org.neo4j.kernel.impl.storageengine.impl.recordstorage.RecordStorageEngine;
import org.neo4j.kernel.impl.store.format.standard.Standard;
import org.neo4j.kernel.impl.store.format.standard.StandardV3_2;
import org.neo4j.kernel.impl.store.format.standard.StandardV3_4;
import org.neo4j.kernel.impl.storemigration.StoreUpgrader;
import org.neo4j.kernel.impl.storemigration.UpgradeNotAllowedByConfigurationException;
import org.neo4j.kernel.internal.GraphDatabaseAPI;
import org.neo4j.test.rule.TestDirectory;

import static org.hamcrest.Matchers.startsWith;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertSame;
import static org.neo4j.graphdb.factory.GraphDatabaseSettings.allow_upgrade;
import static org.neo4j.graphdb.factory.GraphDatabaseSettings.record_format;

public class RecordFormatMigrationIT
{
@Rule
public final TestDirectory testDirectory = TestDirectory.testDirectory();
private File storeDir;

@Before
public void setUp()
{
storeDir = testDirectory.graphDbDir();
}

@Test
public void failToDowngradeFormatWhenUpgradeNotAllowed()
{
GraphDatabaseService database = startDatabaseWithFormat( storeDir, StandardV3_4.NAME );
try ( Transaction transaction = database.beginTx() )
{
Node node = database.createNode();
node.setProperty( "a", "b" );
transaction.success();
}
database.shutdown();
try
{
startDatabaseWithFormat( storeDir, StandardV3_2.NAME );
}
catch ( Throwable t )
{
assertSame( UpgradeNotAllowedByConfigurationException.class, Exceptions.rootCause( t ).getClass() );
}
}

@Test
public void failToDowngradeFormatWheUpgradeAllowed()
{
GraphDatabaseService database = startDatabaseWithFormat( storeDir, StandardV3_4.NAME );
try ( Transaction transaction = database.beginTx() )
{
Node node = database.createNode();
node.setProperty( "a", "b" );
transaction.success();
}
database.shutdown();
try
{
new GraphDatabaseFactory().newEmbeddedDatabaseBuilder( storeDir )
.setConfig( record_format, StandardV3_2.NAME )
.setConfig( allow_upgrade, Settings.TRUE )
.newGraphDatabase();
}
catch ( Throwable t )
{
assertSame( StoreUpgrader.AttemptedDowngradeException.class, Exceptions.rootCause( t ).getClass() );
}
}

@Test
public void skipMigrationIfFormatSpecifiedInConfig() throws Exception
{
File storeDir = testDirectory.graphDbDir();
GraphDatabaseService database = startDatabaseWithFormat( storeDir, StandardV3_2.NAME );
try ( Transaction transaction = database.beginTx() )
{
Expand All @@ -66,7 +123,25 @@ public void skipMigrationIfFormatSpecifiedInConfig() throws Exception
@Test
public void skipMigrationIfStoreFormatNotSpecifiedButIsAvailableInRuntime() throws Exception
{
File storeDir = testDirectory.graphDbDir();
GraphDatabaseService database = startDatabaseWithFormat( storeDir, StandardV3_2.NAME );
try ( Transaction transaction = database.beginTx() )
{
Node node = database.createNode();
node.setProperty( "a", "b" );
transaction.success();
}
database.shutdown();

GraphDatabaseAPI nonUpgradedStore = (GraphDatabaseAPI) new GraphDatabaseFactory()
.newEmbeddedDatabase( storeDir );
RecordStorageEngine storageEngine = nonUpgradedStore.getDependencyResolver().resolveDependency( RecordStorageEngine.class );
assertEquals( StandardV3_2.NAME, storageEngine.testAccessNeoStores().getRecordFormats().name() );
nonUpgradedStore.shutdown();
}

@Test
public void latestRecordNotMigratedWhenFormatBumped()
{
GraphDatabaseService database = startDatabaseWithFormat( storeDir, StandardV3_2.NAME );
try ( Transaction transaction = database.beginTx() )
{
Expand All @@ -78,15 +153,11 @@ public void skipMigrationIfStoreFormatNotSpecifiedButIsAvailableInRuntime() thro

try
{
GraphDatabaseAPI nonUpgradedStore = (GraphDatabaseAPI) new GraphDatabaseFactory().newEmbeddedDatabase( storeDir );
RecordStorageEngine storageEngine = nonUpgradedStore.getDependencyResolver().resolveDependency( RecordStorageEngine.class );
assertEquals( StandardV3_2.NAME, storageEngine.testAccessNeoStores().getRecordFormats().name() );
nonUpgradedStore.shutdown();
startDatabaseWithFormat( storeDir, Standard.LATEST_NAME );
}
catch ( Throwable e )
catch ( Throwable t )
{
assertThat( Exceptions.rootCause( e ).getMessage(), startsWith( "Neo4j cannot be started because the " +
"database files require upgrading and upgrades are disabled in the configuration." ) );
assertSame( UpgradeNotAllowedByConfigurationException.class, Exceptions.rootCause( t ).getClass() );
}
}

Expand Down
6 changes: 1 addition & 5 deletions community/neo4j/src/test/java/migration/Start3_2DbOn3_3AndCreateFusionIndexIT.java
View file
Expand Up @@ -33,7 +33,6 @@
import org.neo4j.graphdb.Node;
import org.neo4j.graphdb.Transaction;
import org.neo4j.graphdb.factory.GraphDatabaseFactory;
import org.neo4j.graphdb.factory.GraphDatabaseSettings;
import org.neo4j.graphdb.schema.IndexDefinition;
import org.neo4j.internal.kernel.api.IndexQuery;
import org.neo4j.io.fs.FileUtils;
Expand All @@ -44,7 +43,6 @@
import org.neo4j.kernel.api.schema.index.IndexDescriptor;
import org.neo4j.kernel.impl.api.KernelStatement;
import org.neo4j.kernel.impl.core.ThreadToStatementContextBridge;
import org.neo4j.kernel.impl.store.format.standard.StandardV3_2;
import org.neo4j.kernel.internal.GraphDatabaseAPI;
import org.neo4j.storageengine.api.StorageStatement;
import org.neo4j.storageengine.api.schema.IndexReader;
Expand Down Expand Up @@ -91,9 +89,7 @@ public void shouldOpen3_2DbAndCreateAndWorkWithSomeFusionIndexes() throws Except
{
// given
File storeDir = unzip( getClass(), ZIP_FILE, directory.absolutePath() );
GraphDatabaseAPI db = (GraphDatabaseAPI) new GraphDatabaseFactory().newEmbeddedDatabaseBuilder( storeDir )
//TODO: should not be required later on
.setConfig( GraphDatabaseSettings.record_format, StandardV3_2.NAME ).newGraphDatabase();
GraphDatabaseAPI db = (GraphDatabaseAPI) new GraphDatabaseFactory().newEmbeddedDatabase( storeDir );
try
{
verifyIndexes( db, LABEL1 );
Expand Down

0 comments on commit 3d31f75

Please sign in to comment.