Skip to content

Commit

Permalink
Use store layout for store file access operations.
Browse files Browse the repository at this point in the history
Hide store_lock file under store layout.
  • Loading branch information
MishaDemianenko committed Aug 21, 2018
1 parent c1a74b8 commit 0e0a77c
Show file tree
Hide file tree
Showing 29 changed files with 152 additions and 132 deletions.
Expand Up @@ -30,6 +30,7 @@
import org.neo4j.commandline.admin.CommandFailed; import org.neo4j.commandline.admin.CommandFailed;
import org.neo4j.io.fs.DefaultFileSystemAbstraction; import org.neo4j.io.fs.DefaultFileSystemAbstraction;
import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.fs.FileSystemAbstraction;
import org.neo4j.io.layout.StoreLayout;
import org.neo4j.kernel.StoreLockException; import org.neo4j.kernel.StoreLockException;
import org.neo4j.kernel.internal.locker.GlobalStoreLocker; import org.neo4j.kernel.internal.locker.GlobalStoreLocker;
import org.neo4j.kernel.internal.locker.StoreLocker; import org.neo4j.kernel.internal.locker.StoreLocker;
Expand Down Expand Up @@ -78,10 +79,10 @@ public static boolean isSameOrChildPath( Path parent, Path candidate )
return normalizedCandidate.startsWith( normalizedParent ); return normalizedCandidate.startsWith( normalizedParent );
} }


public static void checkLock( Path databaseDirectory ) throws CommandFailed public static void checkLock( StoreLayout storeLayout ) throws CommandFailed
{ {
try ( FileSystemAbstraction fileSystem = new DefaultFileSystemAbstraction(); try ( FileSystemAbstraction fileSystem = new DefaultFileSystemAbstraction();
StoreLocker storeLocker = new GlobalStoreLocker( fileSystem, databaseDirectory.getParent().toFile() ) ) StoreLocker storeLocker = new GlobalStoreLocker( fileSystem, storeLayout ) )
{ {
storeLocker.checkLock(); storeLocker.checkLock();
} }
Expand Down
Expand Up @@ -195,7 +195,7 @@ public ImpermanentPlatformModule( File storeDir, Config config, DatabaseInfo dat
@Override @Override
protected StoreLocker createStoreLocker() protected StoreLocker createStoreLocker()
{ {
return new StoreLocker( fileSystem, storeLayout.storeDirectory() ); return new StoreLocker( fileSystem, storeLayout );
} }


@Override @Override
Expand Down
Expand Up @@ -350,7 +350,7 @@ protected FileSystemAbstraction createNewFileSystem()
@Override @Override
protected StoreLocker createStoreLocker() protected StoreLocker createStoreLocker()
{ {
return new StoreLocker( fileSystem, storeLayout.storeDirectory() ); return new StoreLocker( fileSystem, storeLayout );
} }
} }
} }
Expand Down
Expand Up @@ -35,6 +35,7 @@
import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.fs.FileSystemAbstraction;
import org.neo4j.io.fs.OpenMode; import org.neo4j.io.fs.OpenMode;
import org.neo4j.io.fs.StoreChannel; import org.neo4j.io.fs.StoreChannel;
import org.neo4j.io.layout.StoreLayout;
import org.neo4j.kernel.StoreLockException; import org.neo4j.kernel.StoreLockException;
import org.neo4j.kernel.internal.locker.StoreLocker; import org.neo4j.kernel.internal.locker.StoreLocker;
import org.neo4j.test.TestGraphDatabaseFactory; import org.neo4j.test.TestGraphDatabaseFactory;
Expand All @@ -47,7 +48,6 @@
import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertThat; import static org.junit.Assert.assertThat;
import static org.junit.Assert.fail; import static org.junit.Assert.fail;
import static org.neo4j.kernel.internal.locker.StoreLocker.STORE_LOCK_FILENAME;


public class StoreLockerTest public class StoreLockerTest
{ {
Expand All @@ -63,7 +63,7 @@ public void shouldUseAlreadyOpenedFileChannel() throws Exception
CustomChannelFileSystemAbstraction fileSystemAbstraction = CustomChannelFileSystemAbstraction fileSystemAbstraction =
new CustomChannelFileSystemAbstraction( fileSystemRule.get(), channel ); new CustomChannelFileSystemAbstraction( fileSystemRule.get(), channel );
int numberOfCallesToOpen = 0; int numberOfCallesToOpen = 0;
try ( StoreLocker storeLocker = new StoreLocker( fileSystemAbstraction, target.directory( "unused" ) ) ) try ( StoreLocker storeLocker = new StoreLocker( fileSystemAbstraction, target.storeLayout() ) )
{ {
try try
{ {
Expand Down Expand Up @@ -91,7 +91,7 @@ public void shouldUseAlreadyOpenedFileChannel() throws Exception
@Test @Test
public void shouldAllowMultipleCallsToCheckLock() throws Exception public void shouldAllowMultipleCallsToCheckLock() throws Exception
{ {
try ( StoreLocker storeLocker = new StoreLocker( fileSystemRule.get(), target.directory( "unused" ) ) ) try ( StoreLocker storeLocker = new StoreLocker( fileSystemRule.get(), target.storeLayout() ) )
{ {
storeLocker.checkLock(); storeLocker.checkLock();
storeLocker.checkLock(); storeLocker.checkLock();
Expand All @@ -101,12 +101,12 @@ public void shouldAllowMultipleCallsToCheckLock() throws Exception
@Test @Test
public void keepLockWhenOtherTryToTakeLock() throws Exception public void keepLockWhenOtherTryToTakeLock() throws Exception
{ {
File directory = target.directory( "unused" ); StoreLayout storeLayout = target.storeLayout();
DefaultFileSystemAbstraction fileSystemAbstraction = fileSystemRule.get(); DefaultFileSystemAbstraction fileSystemAbstraction = fileSystemRule.get();
StoreLocker storeLocker = new StoreLocker( fileSystemAbstraction, directory ); StoreLocker storeLocker = new StoreLocker( fileSystemAbstraction, storeLayout );
storeLocker.checkLock(); storeLocker.checkLock();


try ( StoreLocker storeLocker1 = new StoreLocker( fileSystemAbstraction, directory ) ) try ( StoreLocker storeLocker1 = new StoreLocker( fileSystemAbstraction, storeLayout ) )
{ {
storeLocker1.checkLock(); storeLocker1.checkLock();
fail(); fail();
Expand All @@ -117,7 +117,7 @@ public void keepLockWhenOtherTryToTakeLock() throws Exception
} }


// Initial locker should still have a valid lock // Initial locker should still have a valid lock
try ( StoreLocker storeLocker1 = new StoreLocker( fileSystemAbstraction, directory ) ) try ( StoreLocker storeLocker1 = new StoreLocker( fileSystemAbstraction, storeLayout ) )
{ {
storeLocker1.checkLock(); storeLocker1.checkLock();
fail(); fail();
Expand All @@ -142,7 +142,7 @@ public boolean fileExists( File fileName )
} }
}; };


try ( StoreLocker storeLocker = new StoreLocker( fileSystemAbstraction, target.directory( "unused" ) ) ) try ( StoreLocker storeLocker = new StoreLocker( fileSystemAbstraction, target.storeLayout() ) )
{ {
storeLocker.checkLock(); storeLocker.checkLock();


Expand All @@ -166,7 +166,7 @@ public boolean fileExists( File fileName )
} }
}; };


try ( StoreLocker storeLocker = new StoreLocker( fileSystemAbstraction, target.directory( "unused" ) ) ) try ( StoreLocker storeLocker = new StoreLocker( fileSystemAbstraction, target.storeLayout() ) )
{ {
storeLocker.checkLock(); storeLocker.checkLock();
// Ok // Ok
Expand All @@ -191,9 +191,8 @@ public boolean fileExists( File fileName )
} }
}; };


File storeDir = target.directory( "unused" ); StoreLayout storeLayout = target.storeLayout();

try ( StoreLocker storeLocker = new StoreLocker( fileSystemAbstraction, storeLayout ) )
try ( StoreLocker storeLocker = new StoreLocker( fileSystemAbstraction, storeDir ) )
{ {
storeLocker.checkLock(); storeLocker.checkLock();
fail(); fail();
Expand All @@ -202,7 +201,7 @@ public boolean fileExists( File fileName )
{ {
String msg = format( "Unable to create path for store dir: %s. " + String msg = format( "Unable to create path for store dir: %s. " +
"Please ensure no other process is using this database, and that " + "Please ensure no other process is using this database, and that " +
"the directory is writable (required even for read-only access)", storeDir ); "the directory is writable (required even for read-only access)", storeLayout );
assertThat( e.getMessage(), is( msg ) ); assertThat( e.getMessage(), is( msg ) );
} }
} }
Expand All @@ -225,9 +224,9 @@ public boolean fileExists( File fileName )
} }
}; };


File storeDir = target.directory( "unused" ); StoreLayout storeLayout = target.storeLayout();


try ( StoreLocker storeLocker = new StoreLocker( fileSystemAbstraction, storeDir ) ) try ( StoreLocker storeLocker = new StoreLocker( fileSystemAbstraction, storeLayout ) )
{ {
storeLocker.checkLock(); storeLocker.checkLock();
fail(); fail();
Expand All @@ -237,7 +236,7 @@ public boolean fileExists( File fileName )
String msg = format( "Unable to obtain lock on store lock file: %s. " + String msg = format( "Unable to obtain lock on store lock file: %s. " +
"Please ensure no other process is using this database, and that the " + "Please ensure no other process is using this database, and that the " +
"directory is writable (required even for read-only access)", "directory is writable (required even for read-only access)",
new File( storeDir, STORE_LOCK_FILENAME ) ); storeLayout.storeLockFile() );
assertThat( e.getMessage(), is( msg ) ); assertThat( e.getMessage(), is( msg ) );
} }
} }
Expand Down Expand Up @@ -267,7 +266,7 @@ public FileLock tryLock()
} }
}; };


try ( StoreLocker storeLocker = new StoreLocker( fileSystemAbstraction, target.directory( "unused" ) ) ) try ( StoreLocker storeLocker = new StoreLocker( fileSystemAbstraction, target.storeLayout() ) )
{ {
storeLocker.checkLock(); storeLocker.checkLock();
fail(); fail();
Expand Down
Expand Up @@ -24,12 +24,13 @@
import org.junit.rules.ExpectedException; import org.junit.rules.ExpectedException;
import org.junit.rules.RuleChain; import org.junit.rules.RuleChain;


import java.io.File;
import java.io.IOException; import java.io.IOException;


import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.graphdb.factory.GraphDatabaseSettings;
import org.neo4j.io.fs.DefaultFileSystemAbstraction; import org.neo4j.io.fs.DefaultFileSystemAbstraction;
import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.fs.FileSystemAbstraction;
import org.neo4j.io.layout.DatabaseLayout;
import org.neo4j.io.layout.StoreLayout;
import org.neo4j.io.pagecache.PageCache; import org.neo4j.io.pagecache.PageCache;
import org.neo4j.io.pagecache.impl.muninn.MuninnPageCache; import org.neo4j.io.pagecache.impl.muninn.MuninnPageCache;
import org.neo4j.kernel.StoreLockException; import org.neo4j.kernel.StoreLockException;
Expand Down Expand Up @@ -77,31 +78,31 @@ public void testHonorsPassedInParams() throws Exception
public void testCreatesStoreLockFile() throws Exception public void testCreatesStoreLockFile() throws Exception
{ {
// Given // Given
File file = testDirectory.databaseDir(); DatabaseLayout databaseLayout = testDirectory.databaseLayout();


// When // When
BatchInserter inserter = BatchInserters.inserter( file.getAbsoluteFile(), fileSystemRule.get() ); BatchInserter inserter = BatchInserters.inserter( databaseLayout.databaseDirectory(), fileSystemRule.get() );


// Then // Then
assertThat( new File( file, StoreLocker.STORE_LOCK_FILENAME ).exists(), equalTo( true ) ); assertThat( databaseLayout.getStoreLayout().storeLockFile().exists(), equalTo( true ) );
inserter.shutdown(); inserter.shutdown();
} }


@Test @Test
public void testFailsOnExistingStoreLockFile() throws IOException public void testFailsOnExistingStoreLockFile() throws IOException
{ {
// Given // Given
File parent = testDirectory.databaseDir(); StoreLayout storeLayout = testDirectory.storeLayout();
try ( FileSystemAbstraction fileSystemAbstraction = new DefaultFileSystemAbstraction(); try ( FileSystemAbstraction fileSystemAbstraction = new DefaultFileSystemAbstraction();
StoreLocker lock = new StoreLocker( fileSystemAbstraction, parent ) ) StoreLocker lock = new StoreLocker( fileSystemAbstraction, storeLayout ) )
{ {
lock.checkLock(); lock.checkLock();


// Then // Then
expected.expect( StoreLockException.class ); expected.expect( StoreLockException.class );
expected.expectMessage( "Unable to obtain lock on store lock file" ); expected.expectMessage( "Unable to obtain lock on store lock file" );
// When // When
BatchInserters.inserter( parent.getAbsoluteFile(), fileSystemAbstraction ); BatchInserters.inserter( storeLayout.databaseLayout( "any" ).databaseDirectory(), fileSystemAbstraction );
} }
} }
} }
Expand Up @@ -20,6 +20,7 @@
package org.neo4j.commandline.dbms; package org.neo4j.commandline.dbms;


import java.io.Closeable; import java.io.Closeable;
import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.nio.file.FileAlreadyExistsException; import java.nio.file.FileAlreadyExistsException;
import java.nio.file.Files; import java.nio.file.Files;
Expand All @@ -32,13 +33,12 @@
import org.neo4j.commandline.admin.CommandFailed; import org.neo4j.commandline.admin.CommandFailed;
import org.neo4j.commandline.admin.IncorrectUsage; import org.neo4j.commandline.admin.IncorrectUsage;
import org.neo4j.commandline.arguments.Arguments; import org.neo4j.commandline.arguments.Arguments;
import org.neo4j.commandline.arguments.common.Database;
import org.neo4j.dbms.archive.Dumper; import org.neo4j.dbms.archive.Dumper;
import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.graphdb.factory.GraphDatabaseSettings;
import org.neo4j.io.layout.DatabaseLayout;
import org.neo4j.kernel.StoreLockException; import org.neo4j.kernel.StoreLockException;
import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.configuration.Config;
import org.neo4j.kernel.impl.util.Validators; import org.neo4j.kernel.impl.util.Validators;
import org.neo4j.kernel.internal.locker.StoreLocker;


import static java.lang.String.format; import static java.lang.String.format;
import static org.neo4j.commandline.Util.canonicalPath; import static org.neo4j.commandline.Util.canonicalPath;
Expand Down Expand Up @@ -72,6 +72,7 @@ public void execute( String[] args ) throws IncorrectUsage, CommandFailed


Config config = buildConfig( database ); Config config = buildConfig( database );
Path databaseDirectory = canonicalPath( getDatabaseDirectory( config ) ); Path databaseDirectory = canonicalPath( getDatabaseDirectory( config ) );
DatabaseLayout databaseLayout = DatabaseLayout.of( databaseDirectory.toFile() );
Path transactionLogsDirectory = canonicalPath( getTransactionalLogsDirectory( config ) ); Path transactionLogsDirectory = canonicalPath( getTransactionalLogsDirectory( config ) );


try try
Expand All @@ -83,9 +84,9 @@ public void execute( String[] args ) throws IncorrectUsage, CommandFailed
throw new CommandFailed( "database does not exist: " + database, e ); throw new CommandFailed( "database does not exist: " + database, e );
} }


try ( Closeable ignored = StoreLockChecker.check( databaseDirectory ) ) try ( Closeable ignored = StoreLockChecker.check( databaseLayout.getStoreLayout() ) )
{ {
dump( database, databaseDirectory, transactionLogsDirectory, archive ); dump( database, databaseLayout, transactionLogsDirectory, archive );
} }
catch ( StoreLockException e ) catch ( StoreLockException e )
{ {
Expand All @@ -101,12 +102,12 @@ public void execute( String[] args ) throws IncorrectUsage, CommandFailed
} }
} }


private Path getDatabaseDirectory( Config config ) private static Path getDatabaseDirectory( Config config )
{ {
return config.get( database_path ).toPath(); return config.get( database_path ).toPath();
} }


private Path getTransactionalLogsDirectory( Config config ) private static Path getTransactionalLogsDirectory( Config config )
{ {
return config.get( logical_logs_location ).toPath(); return config.get( logical_logs_location ).toPath();
} }
Expand All @@ -120,25 +121,27 @@ private Config buildConfig( String databaseName )
.build(); .build();
} }


private Path calculateArchive( String database, Path to ) private static Path calculateArchive( String database, Path to )
{ {
return Files.isDirectory( to ) ? to.resolve( database + ".dump" ) : to; return Files.isDirectory( to ) ? to.resolve( database + ".dump" ) : to;
} }


private void dump( String database, Path databaseDirectory, Path transactionalLogsDirectory, Path archive ) private void dump( String database, DatabaseLayout databaseLayout, Path transactionalLogsDirectory, Path archive )
throws CommandFailed throws CommandFailed
{ {
Path databasePath = databaseLayout.databaseDirectory().toPath();
try try
{ {
dumper.dump( databaseDirectory, transactionalLogsDirectory, archive, this::isStoreLock ); File storeLockFile = databaseLayout.getStoreLayout().storeLockFile();
dumper.dump( databasePath, transactionalLogsDirectory, archive, path -> Objects.equals( path.getFileName().toString(), storeLockFile.getName() ) );
} }
catch ( FileAlreadyExistsException e ) catch ( FileAlreadyExistsException e )
{ {
throw new CommandFailed( "archive already exists: " + e.getMessage(), e ); throw new CommandFailed( "archive already exists: " + e.getMessage(), e );
} }
catch ( NoSuchFileException e ) catch ( NoSuchFileException e )
{ {
if ( Paths.get( e.getMessage() ).toAbsolutePath().equals( databaseDirectory.toAbsolutePath() ) ) if ( Paths.get( e.getMessage() ).toAbsolutePath().equals( databasePath ) )
{ {
throw new CommandFailed( "database does not exist: " + database, e ); throw new CommandFailed( "database does not exist: " + database, e );
} }
Expand All @@ -150,11 +153,6 @@ private void dump( String database, Path databaseDirectory, Path transactionalLo
} }
} }


private boolean isStoreLock( Path path )
{
return Objects.equals( path.getFileName().toString(), StoreLocker.STORE_LOCK_FILENAME );
}

private void wrapIOException( IOException e ) throws CommandFailed private void wrapIOException( IOException e ) throws CommandFailed
{ {
throw new CommandFailed( throw new CommandFailed(
Expand Down
Expand Up @@ -31,12 +31,12 @@
import org.neo4j.commandline.admin.IncorrectUsage; import org.neo4j.commandline.admin.IncorrectUsage;
import org.neo4j.commandline.arguments.Arguments; import org.neo4j.commandline.arguments.Arguments;
import org.neo4j.commandline.arguments.OptionalBooleanArg; import org.neo4j.commandline.arguments.OptionalBooleanArg;
import org.neo4j.commandline.arguments.common.Database;
import org.neo4j.commandline.arguments.common.MandatoryCanonicalPath; import org.neo4j.commandline.arguments.common.MandatoryCanonicalPath;
import org.neo4j.dbms.archive.IncorrectFormat; import org.neo4j.dbms.archive.IncorrectFormat;
import org.neo4j.dbms.archive.Loader; import org.neo4j.dbms.archive.Loader;
import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.graphdb.factory.GraphDatabaseSettings;
import org.neo4j.io.fs.FileUtils; import org.neo4j.io.fs.FileUtils;
import org.neo4j.io.layout.DatabaseLayout;
import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.configuration.Config;


import static java.util.Objects.requireNonNull; import static java.util.Objects.requireNonNull;
Expand Down Expand Up @@ -112,7 +112,7 @@ private void deleteIfNecessary( Path databaseDirectory, Path transactionLogsDire
{ {
if ( force ) if ( force )
{ {
checkLock( databaseDirectory ); checkLock( DatabaseLayout.of( databaseDirectory.toFile() ).getStoreLayout() );
FileUtils.deletePathRecursively( databaseDirectory ); FileUtils.deletePathRecursively( databaseDirectory );
if ( !isSameOrChildPath( databaseDirectory, transactionLogsDirectory ) ) if ( !isSameOrChildPath( databaseDirectory, transactionLogsDirectory ) )
{ {
Expand Down
Expand Up @@ -20,14 +20,14 @@
package org.neo4j.commandline.dbms; package org.neo4j.commandline.dbms;


import java.io.Closeable; import java.io.Closeable;
import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;


import org.neo4j.io.IOUtils; import org.neo4j.io.IOUtils;
import org.neo4j.io.fs.DefaultFileSystemAbstraction; import org.neo4j.io.fs.DefaultFileSystemAbstraction;
import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.fs.FileSystemAbstraction;
import org.neo4j.io.layout.StoreLayout;
import org.neo4j.kernel.StoreLockException; import org.neo4j.kernel.StoreLockException;
import org.neo4j.kernel.internal.locker.GlobalStoreLocker; import org.neo4j.kernel.internal.locker.GlobalStoreLocker;
import org.neo4j.kernel.internal.locker.StoreLocker; import org.neo4j.kernel.internal.locker.StoreLocker;
Expand All @@ -38,29 +38,29 @@ class StoreLockChecker implements Closeable
private final FileSystemAbstraction fileSystem; private final FileSystemAbstraction fileSystem;
private final StoreLocker storeLocker; private final StoreLocker storeLocker;


private StoreLockChecker( FileSystemAbstraction fileSystem, File storeDirectory ) private StoreLockChecker( FileSystemAbstraction fileSystem, StoreLayout storeLayout )
{ {
this.fileSystem = fileSystem; this.fileSystem = fileSystem;
this.storeLocker = new GlobalStoreLocker( fileSystem, storeDirectory ); this.storeLocker = new GlobalStoreLocker( fileSystem, storeLayout );
} }


/** /**
* Create store lock checker with lock on a provided path if it exists and writable * Create store lock checker with lock on a provided store layout if it exists and writable
* @param databaseDirectory database path * @param storeLayout store layout to check
* @return lock checker or empty closeable in case if path does not exists or is not writable * @return lock checker or empty closeable in case if path does not exists or is not writable
* @throws CannotWriteException * @throws CannotWriteException
* *
* @see StoreLocker * @see StoreLocker
* @see Files * @see Files
*/ */
static Closeable check( Path databaseDirectory ) throws CannotWriteException static Closeable check( StoreLayout storeLayout ) throws CannotWriteException
{ {
Path lockFile = databaseDirectory.resolve( StoreLocker.STORE_LOCK_FILENAME ); Path lockFile = storeLayout.storeLockFile().toPath();
if ( Files.exists( lockFile ) ) if ( Files.exists( lockFile ) )
{ {
if ( Files.isWritable( lockFile ) ) if ( Files.isWritable( lockFile ) )
{ {
StoreLockChecker storeLocker = new StoreLockChecker( new DefaultFileSystemAbstraction(), databaseDirectory.toFile() ); StoreLockChecker storeLocker = new StoreLockChecker( new DefaultFileSystemAbstraction(), storeLayout );
try try
{ {
storeLocker.checkLock(); storeLocker.checkLock();
Expand Down

0 comments on commit 0e0a77c

Please sign in to comment.