Skip to content

Commit

Permalink
Use store layout for store file access operations.
Browse files Browse the repository at this point in the history
Hide store_lock file under store layout.
  • Loading branch information
MishaDemianenko committed Aug 21, 2018
1 parent c1a74b8 commit 0e0a77c
Show file tree
Hide file tree
Showing 29 changed files with 152 additions and 132 deletions.
Expand Up @@ -30,6 +30,7 @@
import org.neo4j.commandline.admin.CommandFailed;
import org.neo4j.io.fs.DefaultFileSystemAbstraction;
import org.neo4j.io.fs.FileSystemAbstraction;
import org.neo4j.io.layout.StoreLayout;
import org.neo4j.kernel.StoreLockException;
import org.neo4j.kernel.internal.locker.GlobalStoreLocker;
import org.neo4j.kernel.internal.locker.StoreLocker;
Expand Down Expand Up @@ -78,10 +79,10 @@ public static boolean isSameOrChildPath( Path parent, Path candidate )
return normalizedCandidate.startsWith( normalizedParent );
}

public static void checkLock( Path databaseDirectory ) throws CommandFailed
public static void checkLock( StoreLayout storeLayout ) throws CommandFailed
{
try ( FileSystemAbstraction fileSystem = new DefaultFileSystemAbstraction();
StoreLocker storeLocker = new GlobalStoreLocker( fileSystem, databaseDirectory.getParent().toFile() ) )
StoreLocker storeLocker = new GlobalStoreLocker( fileSystem, storeLayout ) )
{
storeLocker.checkLock();
}
Expand Down
Expand Up @@ -195,7 +195,7 @@ public ImpermanentPlatformModule( File storeDir, Config config, DatabaseInfo dat
@Override
protected StoreLocker createStoreLocker()
{
return new StoreLocker( fileSystem, storeLayout.storeDirectory() );
return new StoreLocker( fileSystem, storeLayout );
}

@Override
Expand Down
Expand Up @@ -350,7 +350,7 @@ protected FileSystemAbstraction createNewFileSystem()
@Override
protected StoreLocker createStoreLocker()
{
return new StoreLocker( fileSystem, storeLayout.storeDirectory() );
return new StoreLocker( fileSystem, storeLayout );
}
}
}
Expand Down
Expand Up @@ -35,6 +35,7 @@
import org.neo4j.io.fs.FileSystemAbstraction;
import org.neo4j.io.fs.OpenMode;
import org.neo4j.io.fs.StoreChannel;
import org.neo4j.io.layout.StoreLayout;
import org.neo4j.kernel.StoreLockException;
import org.neo4j.kernel.internal.locker.StoreLocker;
import org.neo4j.test.TestGraphDatabaseFactory;
Expand All @@ -47,7 +48,6 @@
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.fail;
import static org.neo4j.kernel.internal.locker.StoreLocker.STORE_LOCK_FILENAME;

public class StoreLockerTest
{
Expand All @@ -63,7 +63,7 @@ public void shouldUseAlreadyOpenedFileChannel() throws Exception
CustomChannelFileSystemAbstraction fileSystemAbstraction =
new CustomChannelFileSystemAbstraction( fileSystemRule.get(), channel );
int numberOfCallesToOpen = 0;
try ( StoreLocker storeLocker = new StoreLocker( fileSystemAbstraction, target.directory( "unused" ) ) )
try ( StoreLocker storeLocker = new StoreLocker( fileSystemAbstraction, target.storeLayout() ) )
{
try
{
Expand Down Expand Up @@ -91,7 +91,7 @@ public void shouldUseAlreadyOpenedFileChannel() throws Exception
@Test
public void shouldAllowMultipleCallsToCheckLock() throws Exception
{
try ( StoreLocker storeLocker = new StoreLocker( fileSystemRule.get(), target.directory( "unused" ) ) )
try ( StoreLocker storeLocker = new StoreLocker( fileSystemRule.get(), target.storeLayout() ) )
{
storeLocker.checkLock();
storeLocker.checkLock();
Expand All @@ -101,12 +101,12 @@ public void shouldAllowMultipleCallsToCheckLock() throws Exception
@Test
public void keepLockWhenOtherTryToTakeLock() throws Exception
{
File directory = target.directory( "unused" );
StoreLayout storeLayout = target.storeLayout();
DefaultFileSystemAbstraction fileSystemAbstraction = fileSystemRule.get();
StoreLocker storeLocker = new StoreLocker( fileSystemAbstraction, directory );
StoreLocker storeLocker = new StoreLocker( fileSystemAbstraction, storeLayout );
storeLocker.checkLock();

try ( StoreLocker storeLocker1 = new StoreLocker( fileSystemAbstraction, directory ) )
try ( StoreLocker storeLocker1 = new StoreLocker( fileSystemAbstraction, storeLayout ) )
{
storeLocker1.checkLock();
fail();
Expand All @@ -117,7 +117,7 @@ public void keepLockWhenOtherTryToTakeLock() throws Exception
}

// Initial locker should still have a valid lock
try ( StoreLocker storeLocker1 = new StoreLocker( fileSystemAbstraction, directory ) )
try ( StoreLocker storeLocker1 = new StoreLocker( fileSystemAbstraction, storeLayout ) )
{
storeLocker1.checkLock();
fail();
Expand All @@ -142,7 +142,7 @@ public boolean fileExists( File fileName )
}
};

try ( StoreLocker storeLocker = new StoreLocker( fileSystemAbstraction, target.directory( "unused" ) ) )
try ( StoreLocker storeLocker = new StoreLocker( fileSystemAbstraction, target.storeLayout() ) )
{
storeLocker.checkLock();

Expand All @@ -166,7 +166,7 @@ public boolean fileExists( File fileName )
}
};

try ( StoreLocker storeLocker = new StoreLocker( fileSystemAbstraction, target.directory( "unused" ) ) )
try ( StoreLocker storeLocker = new StoreLocker( fileSystemAbstraction, target.storeLayout() ) )
{
storeLocker.checkLock();
// Ok
Expand All @@ -191,9 +191,8 @@ public boolean fileExists( File fileName )
}
};

File storeDir = target.directory( "unused" );

try ( StoreLocker storeLocker = new StoreLocker( fileSystemAbstraction, storeDir ) )
StoreLayout storeLayout = target.storeLayout();
try ( StoreLocker storeLocker = new StoreLocker( fileSystemAbstraction, storeLayout ) )
{
storeLocker.checkLock();
fail();
Expand All @@ -202,7 +201,7 @@ public boolean fileExists( File fileName )
{
String msg = format( "Unable to create path for store dir: %s. " +
"Please ensure no other process is using this database, and that " +
"the directory is writable (required even for read-only access)", storeDir );
"the directory is writable (required even for read-only access)", storeLayout );
assertThat( e.getMessage(), is( msg ) );
}
}
Expand All @@ -225,9 +224,9 @@ public boolean fileExists( File fileName )
}
};

File storeDir = target.directory( "unused" );
StoreLayout storeLayout = target.storeLayout();

try ( StoreLocker storeLocker = new StoreLocker( fileSystemAbstraction, storeDir ) )
try ( StoreLocker storeLocker = new StoreLocker( fileSystemAbstraction, storeLayout ) )
{
storeLocker.checkLock();
fail();
Expand All @@ -237,7 +236,7 @@ public boolean fileExists( File fileName )
String msg = format( "Unable to obtain lock on store lock file: %s. " +
"Please ensure no other process is using this database, and that the " +
"directory is writable (required even for read-only access)",
new File( storeDir, STORE_LOCK_FILENAME ) );
storeLayout.storeLockFile() );
assertThat( e.getMessage(), is( msg ) );
}
}
Expand Down Expand Up @@ -267,7 +266,7 @@ public FileLock tryLock()
}
};

try ( StoreLocker storeLocker = new StoreLocker( fileSystemAbstraction, target.directory( "unused" ) ) )
try ( StoreLocker storeLocker = new StoreLocker( fileSystemAbstraction, target.storeLayout() ) )
{
storeLocker.checkLock();
fail();
Expand Down
Expand Up @@ -24,12 +24,13 @@
import org.junit.rules.ExpectedException;
import org.junit.rules.RuleChain;

import java.io.File;
import java.io.IOException;

import org.neo4j.graphdb.factory.GraphDatabaseSettings;
import org.neo4j.io.fs.DefaultFileSystemAbstraction;
import org.neo4j.io.fs.FileSystemAbstraction;
import org.neo4j.io.layout.DatabaseLayout;
import org.neo4j.io.layout.StoreLayout;
import org.neo4j.io.pagecache.PageCache;
import org.neo4j.io.pagecache.impl.muninn.MuninnPageCache;
import org.neo4j.kernel.StoreLockException;
Expand Down Expand Up @@ -77,31 +78,31 @@ public void testHonorsPassedInParams() throws Exception
public void testCreatesStoreLockFile() throws Exception
{
// Given
File file = testDirectory.databaseDir();
DatabaseLayout databaseLayout = testDirectory.databaseLayout();

// When
BatchInserter inserter = BatchInserters.inserter( file.getAbsoluteFile(), fileSystemRule.get() );
BatchInserter inserter = BatchInserters.inserter( databaseLayout.databaseDirectory(), fileSystemRule.get() );

// Then
assertThat( new File( file, StoreLocker.STORE_LOCK_FILENAME ).exists(), equalTo( true ) );
assertThat( databaseLayout.getStoreLayout().storeLockFile().exists(), equalTo( true ) );
inserter.shutdown();
}

@Test
public void testFailsOnExistingStoreLockFile() throws IOException
{
// Given
File parent = testDirectory.databaseDir();
StoreLayout storeLayout = testDirectory.storeLayout();
try ( FileSystemAbstraction fileSystemAbstraction = new DefaultFileSystemAbstraction();
StoreLocker lock = new StoreLocker( fileSystemAbstraction, parent ) )
StoreLocker lock = new StoreLocker( fileSystemAbstraction, storeLayout ) )
{
lock.checkLock();

// Then
expected.expect( StoreLockException.class );
expected.expectMessage( "Unable to obtain lock on store lock file" );
// When
BatchInserters.inserter( parent.getAbsoluteFile(), fileSystemAbstraction );
BatchInserters.inserter( storeLayout.databaseLayout( "any" ).databaseDirectory(), fileSystemAbstraction );
}
}
}
Expand Up @@ -20,6 +20,7 @@
package org.neo4j.commandline.dbms;

import java.io.Closeable;
import java.io.File;
import java.io.IOException;
import java.nio.file.FileAlreadyExistsException;
import java.nio.file.Files;
Expand All @@ -32,13 +33,12 @@
import org.neo4j.commandline.admin.CommandFailed;
import org.neo4j.commandline.admin.IncorrectUsage;
import org.neo4j.commandline.arguments.Arguments;
import org.neo4j.commandline.arguments.common.Database;
import org.neo4j.dbms.archive.Dumper;
import org.neo4j.graphdb.factory.GraphDatabaseSettings;
import org.neo4j.io.layout.DatabaseLayout;
import org.neo4j.kernel.StoreLockException;
import org.neo4j.kernel.configuration.Config;
import org.neo4j.kernel.impl.util.Validators;
import org.neo4j.kernel.internal.locker.StoreLocker;

import static java.lang.String.format;
import static org.neo4j.commandline.Util.canonicalPath;
Expand Down Expand Up @@ -72,6 +72,7 @@ public void execute( String[] args ) throws IncorrectUsage, CommandFailed

Config config = buildConfig( database );
Path databaseDirectory = canonicalPath( getDatabaseDirectory( config ) );
DatabaseLayout databaseLayout = DatabaseLayout.of( databaseDirectory.toFile() );
Path transactionLogsDirectory = canonicalPath( getTransactionalLogsDirectory( config ) );

try
Expand All @@ -83,9 +84,9 @@ public void execute( String[] args ) throws IncorrectUsage, CommandFailed
throw new CommandFailed( "database does not exist: " + database, e );
}

try ( Closeable ignored = StoreLockChecker.check( databaseDirectory ) )
try ( Closeable ignored = StoreLockChecker.check( databaseLayout.getStoreLayout() ) )
{
dump( database, databaseDirectory, transactionLogsDirectory, archive );
dump( database, databaseLayout, transactionLogsDirectory, archive );
}
catch ( StoreLockException e )
{
Expand All @@ -101,12 +102,12 @@ public void execute( String[] args ) throws IncorrectUsage, CommandFailed
}
}

private Path getDatabaseDirectory( Config config )
private static Path getDatabaseDirectory( Config config )
{
return config.get( database_path ).toPath();
}

private Path getTransactionalLogsDirectory( Config config )
private static Path getTransactionalLogsDirectory( Config config )
{
return config.get( logical_logs_location ).toPath();
}
Expand All @@ -120,25 +121,27 @@ private Config buildConfig( String databaseName )
.build();
}

private Path calculateArchive( String database, Path to )
private static Path calculateArchive( String database, Path to )
{
return Files.isDirectory( to ) ? to.resolve( database + ".dump" ) : to;
}

private void dump( String database, Path databaseDirectory, Path transactionalLogsDirectory, Path archive )
private void dump( String database, DatabaseLayout databaseLayout, Path transactionalLogsDirectory, Path archive )
throws CommandFailed
{
Path databasePath = databaseLayout.databaseDirectory().toPath();
try
{
dumper.dump( databaseDirectory, transactionalLogsDirectory, archive, this::isStoreLock );
File storeLockFile = databaseLayout.getStoreLayout().storeLockFile();
dumper.dump( databasePath, transactionalLogsDirectory, archive, path -> Objects.equals( path.getFileName().toString(), storeLockFile.getName() ) );
}
catch ( FileAlreadyExistsException e )
{
throw new CommandFailed( "archive already exists: " + e.getMessage(), e );
}
catch ( NoSuchFileException e )
{
if ( Paths.get( e.getMessage() ).toAbsolutePath().equals( databaseDirectory.toAbsolutePath() ) )
if ( Paths.get( e.getMessage() ).toAbsolutePath().equals( databasePath ) )
{
throw new CommandFailed( "database does not exist: " + database, e );
}
Expand All @@ -150,11 +153,6 @@ private void dump( String database, Path databaseDirectory, Path transactionalLo
}
}

private boolean isStoreLock( Path path )
{
return Objects.equals( path.getFileName().toString(), StoreLocker.STORE_LOCK_FILENAME );
}

private void wrapIOException( IOException e ) throws CommandFailed
{
throw new CommandFailed(
Expand Down
Expand Up @@ -31,12 +31,12 @@
import org.neo4j.commandline.admin.IncorrectUsage;
import org.neo4j.commandline.arguments.Arguments;
import org.neo4j.commandline.arguments.OptionalBooleanArg;
import org.neo4j.commandline.arguments.common.Database;
import org.neo4j.commandline.arguments.common.MandatoryCanonicalPath;
import org.neo4j.dbms.archive.IncorrectFormat;
import org.neo4j.dbms.archive.Loader;
import org.neo4j.graphdb.factory.GraphDatabaseSettings;
import org.neo4j.io.fs.FileUtils;
import org.neo4j.io.layout.DatabaseLayout;
import org.neo4j.kernel.configuration.Config;

import static java.util.Objects.requireNonNull;
Expand Down Expand Up @@ -112,7 +112,7 @@ private void deleteIfNecessary( Path databaseDirectory, Path transactionLogsDire
{
if ( force )
{
checkLock( databaseDirectory );
checkLock( DatabaseLayout.of( databaseDirectory.toFile() ).getStoreLayout() );
FileUtils.deletePathRecursively( databaseDirectory );
if ( !isSameOrChildPath( databaseDirectory, transactionLogsDirectory ) )
{
Expand Down
Expand Up @@ -20,14 +20,14 @@
package org.neo4j.commandline.dbms;

import java.io.Closeable;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;

import org.neo4j.io.IOUtils;
import org.neo4j.io.fs.DefaultFileSystemAbstraction;
import org.neo4j.io.fs.FileSystemAbstraction;
import org.neo4j.io.layout.StoreLayout;
import org.neo4j.kernel.StoreLockException;
import org.neo4j.kernel.internal.locker.GlobalStoreLocker;
import org.neo4j.kernel.internal.locker.StoreLocker;
Expand All @@ -38,29 +38,29 @@ class StoreLockChecker implements Closeable
private final FileSystemAbstraction fileSystem;
private final StoreLocker storeLocker;

private StoreLockChecker( FileSystemAbstraction fileSystem, File storeDirectory )
private StoreLockChecker( FileSystemAbstraction fileSystem, StoreLayout storeLayout )
{
this.fileSystem = fileSystem;
this.storeLocker = new GlobalStoreLocker( fileSystem, storeDirectory );
this.storeLocker = new GlobalStoreLocker( fileSystem, storeLayout );
}

/**
* Create store lock checker with lock on a provided path if it exists and writable
* @param databaseDirectory database path
* Create store lock checker with lock on a provided store layout if it exists and writable
* @param storeLayout store layout to check
* @return lock checker or empty closeable in case if path does not exists or is not writable
* @throws CannotWriteException
*
* @see StoreLocker
* @see Files
*/
static Closeable check( Path databaseDirectory ) throws CannotWriteException
static Closeable check( StoreLayout storeLayout ) throws CannotWriteException
{
Path lockFile = databaseDirectory.resolve( StoreLocker.STORE_LOCK_FILENAME );
Path lockFile = storeLayout.storeLockFile().toPath();
if ( Files.exists( lockFile ) )
{
if ( Files.isWritable( lockFile ) )
{
StoreLockChecker storeLocker = new StoreLockChecker( new DefaultFileSystemAbstraction(), databaseDirectory.toFile() );
StoreLockChecker storeLocker = new StoreLockChecker( new DefaultFileSystemAbstraction(), storeLayout );
try
{
storeLocker.checkLock();
Expand Down

0 comments on commit 0e0a77c

Please sign in to comment.