From ec29b7cb30775e80f7c29b418f6773a371707ab3 Mon Sep 17 00:00:00 2001 From: fickludd Date: Tue, 6 Sep 2016 15:15:45 +0200 Subject: [PATCH] Enriched FileSystemAbstraction * renamed renameFile to move and added CopyOptions * added lastModifiedTime --- .../io/fs/DefaultFileSystemAbstraction.java | 15 +++++- .../io/fs/DelegateFileSystemAbstraction.java | 17 +++++-- .../neo4j/io/fs/FileSystemAbstraction.java | 5 +- .../fs/AdversarialFileSystemAbstraction.java | 13 ++++- .../DelegatingFileSystemAbstraction.java | 12 ++++- .../EphemeralFileSystemAbstraction.java | 51 ++++++++++++++++--- .../mockfs/LimitedFilesystemAbstraction.java | 5 +- .../SelectiveFileSystemAbstraction.java | 11 +++- .../kernel/impl/index/IndexConfigStore.java | 4 +- .../storemigration/legacylogs/LegacyLogs.java | 2 +- .../java/org/neo4j/test/LogTestUtils.java | 2 +- .../RotatingFileOutputStreamSupplier.java | 4 +- .../RotatingFileOutputStreamSupplierTest.java | 2 +- 13 files changed, 116 insertions(+), 27 deletions(-) diff --git a/community/io/src/main/java/org/neo4j/io/fs/DefaultFileSystemAbstraction.java b/community/io/src/main/java/org/neo4j/io/fs/DefaultFileSystemAbstraction.java index b8f1736ebd8c1..326506afda408 100644 --- a/community/io/src/main/java/org/neo4j/io/fs/DefaultFileSystemAbstraction.java +++ b/community/io/src/main/java/org/neo4j/io/fs/DefaultFileSystemAbstraction.java @@ -33,6 +33,8 @@ import java.io.Writer; import java.nio.channels.FileChannel; import java.nio.charset.Charset; +import java.nio.file.CopyOption; +import java.nio.file.Files; import java.util.HashMap; import java.util.Map; import java.util.function.Function; @@ -133,8 +135,13 @@ public void deleteRecursively( File directory ) throws IOException } @Override - public boolean renameFile( File from, File to ) throws IOException + public boolean move( File from, File to, CopyOption... copyOptions ) throws IOException { + if ( copyOptions.length > 0 ) + { + Files.move( from.toPath(), to.toPath(), copyOptions ); + return true; // will throw if failure + } return FileUtils.renameFile( from, to ); } @@ -195,6 +202,12 @@ public void truncate( File path, long size ) throws IOException FileUtils.truncateFile( path, size ); } + @Override + public long lastModifiedTime( File file ) + { + return file.lastModified(); + } + protected StoreFileChannel getStoreFileChannel( FileChannel channel ) { return new StoreFileChannel( channel ); diff --git a/community/io/src/main/java/org/neo4j/io/fs/DelegateFileSystemAbstraction.java b/community/io/src/main/java/org/neo4j/io/fs/DelegateFileSystemAbstraction.java index 49886f193f6aa..c6959202ca353 100644 --- a/community/io/src/main/java/org/neo4j/io/fs/DelegateFileSystemAbstraction.java +++ b/community/io/src/main/java/org/neo4j/io/fs/DelegateFileSystemAbstraction.java @@ -30,6 +30,7 @@ import java.io.Writer; import java.nio.channels.FileChannel; import java.nio.charset.Charset; +import java.nio.file.CopyOption; import java.nio.file.DirectoryStream; import java.nio.file.FileSystem; import java.nio.file.Files; @@ -37,9 +38,13 @@ import java.nio.file.StandardCopyOption; import java.util.HashMap; import java.util.Map; +import java.util.concurrent.TimeUnit; import java.util.function.Function; import java.util.stream.Stream; +import static java.nio.file.StandardCopyOption.ATOMIC_MOVE; +import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; + /** * This FileSystemAbstract implementation delegates all calls to a given {@link FileSystem} implementation. * This is useful for testing with arbitrary 3rd party file systems, such as Jimfs. @@ -165,9 +170,9 @@ public void deleteRecursively( File directory ) throws IOException } @Override - public boolean renameFile( File from, File to ) throws IOException + public boolean move( File from, File to, CopyOption... copyOptions ) throws IOException { - Files.move( path( from ), path( to ) ); + Files.move( path( from ), path( to ), copyOptions ); return true; } @@ -241,7 +246,7 @@ private void copyRecursively( Path source, Path target ) throws IOException else { Files.copy( sourcePath, targetPath, - StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.COPY_ATTRIBUTES ); + REPLACE_EXISTING, StandardCopyOption.COPY_ATTRIBUTES ); } } } @@ -271,4 +276,10 @@ public void truncate( File path, long size ) throws IOException channel.truncate( size ); } } + + @Override + public long lastModifiedTime( File file ) throws IOException + { + return Files.getLastModifiedTime( path( file ) ).toMillis(); + } } diff --git a/community/io/src/main/java/org/neo4j/io/fs/FileSystemAbstraction.java b/community/io/src/main/java/org/neo4j/io/fs/FileSystemAbstraction.java index 03f2e55580208..34a98e02cccde 100644 --- a/community/io/src/main/java/org/neo4j/io/fs/FileSystemAbstraction.java +++ b/community/io/src/main/java/org/neo4j/io/fs/FileSystemAbstraction.java @@ -28,6 +28,7 @@ import java.io.Reader; import java.io.Writer; import java.nio.charset.Charset; +import java.nio.file.CopyOption; import java.util.function.Function; import java.util.zip.ZipOutputStream; @@ -57,7 +58,7 @@ public interface FileSystemAbstraction void deleteRecursively( File directory ) throws IOException; - boolean renameFile( File from, File to ) throws IOException; + boolean move( File from, File to, CopyOption... copyOptions ) throws IOException; File[] listFiles( File directory ); @@ -75,6 +76,8 @@ public interface FileSystemAbstraction void truncate( File path, long size ) throws IOException; + long lastModifiedTime( File file ) throws IOException; + interface ThirdPartyFileSystem extends Closeable { void close(); diff --git a/community/io/src/test/java/org/neo4j/adversaries/fs/AdversarialFileSystemAbstraction.java b/community/io/src/test/java/org/neo4j/adversaries/fs/AdversarialFileSystemAbstraction.java index 5846026f9cbe2..9d34fd5067cfb 100644 --- a/community/io/src/test/java/org/neo4j/adversaries/fs/AdversarialFileSystemAbstraction.java +++ b/community/io/src/test/java/org/neo4j/adversaries/fs/AdversarialFileSystemAbstraction.java @@ -32,6 +32,7 @@ import java.lang.reflect.Method; import java.lang.reflect.Proxy; import java.nio.charset.Charset; +import java.nio.file.CopyOption; import java.util.HashMap; import java.util.Map; import java.util.function.Function; @@ -68,10 +69,10 @@ public StoreChannel open( File fileName, String mode ) throws IOException return AdversarialFileChannel.wrap( (StoreFileChannel) delegate.open( fileName, mode ), adversary ); } - public boolean renameFile( File from, File to ) throws IOException + public boolean move( File from, File to, CopyOption... copyOptions ) throws IOException { adversary.injectFailure( FileNotFoundException.class, SecurityException.class ); - return delegate.renameFile( from, to ); + return delegate.move( from, to, copyOptions ); } public OutputStream openAsOutputStream( File fileName, boolean append ) throws IOException @@ -204,6 +205,14 @@ public void truncate( File path, long size ) throws IOException delegate.truncate( path, size ); } + @Override + public long lastModifiedTime( File file ) throws IOException + { + adversary.injectFailure( FileNotFoundException.class, IOException.class, SecurityException.class, + NullPointerException.class ); + return delegate.lastModifiedTime( file ); + } + private ThirdPartyFileSystem adversarialProxy( final ThirdPartyFileSystem fileSystem, Class clazz ) diff --git a/community/io/src/test/java/org/neo4j/graphdb/mockfs/DelegatingFileSystemAbstraction.java b/community/io/src/test/java/org/neo4j/graphdb/mockfs/DelegatingFileSystemAbstraction.java index aeb8ccff08189..ae5d13ee519c7 100644 --- a/community/io/src/test/java/org/neo4j/graphdb/mockfs/DelegatingFileSystemAbstraction.java +++ b/community/io/src/test/java/org/neo4j/graphdb/mockfs/DelegatingFileSystemAbstraction.java @@ -19,6 +19,7 @@ */ package org.neo4j.graphdb.mockfs; +import java.awt.image.ConvolveOp; import java.io.File; import java.io.FilenameFilter; import java.io.IOException; @@ -27,6 +28,7 @@ import java.io.Reader; import java.io.Writer; import java.nio.charset.Charset; +import java.nio.file.CopyOption; import java.util.function.Function; import org.neo4j.io.fs.FileSystemAbstraction; @@ -79,9 +81,15 @@ public void truncate( File path, long size ) throws IOException } @Override - public boolean renameFile( File from, File to ) throws IOException + public long lastModifiedTime( File file ) throws IOException { - return delegate.renameFile( from, to ); + return delegate.lastModifiedTime( file ); + } + + @Override + public boolean move( File from, File to, CopyOption... copyOptions ) throws IOException + { + return delegate.move( from, to, copyOptions ); } @Override diff --git a/community/io/src/test/java/org/neo4j/graphdb/mockfs/EphemeralFileSystemAbstraction.java b/community/io/src/test/java/org/neo4j/graphdb/mockfs/EphemeralFileSystemAbstraction.java index 6653812f00ece..a8f736fb220d8 100644 --- a/community/io/src/test/java/org/neo4j/graphdb/mockfs/EphemeralFileSystemAbstraction.java +++ b/community/io/src/test/java/org/neo4j/graphdb/mockfs/EphemeralFileSystemAbstraction.java @@ -40,6 +40,9 @@ import java.nio.channels.ReadableByteChannel; import java.nio.channels.WritableByteChannel; import java.nio.charset.Charset; +import java.nio.file.CopyOption; +import java.nio.file.StandardCopyOption; +import java.time.Clock; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -70,6 +73,8 @@ public class EphemeralFileSystemAbstraction implements FileSystemAbstraction { + private Clock clock; + interface Positionable { long pos(); @@ -84,6 +89,12 @@ interface Positionable public EphemeralFileSystemAbstraction() { + this( Clock.systemUTC() ); + } + + public EphemeralFileSystemAbstraction( Clock clock ) + { + this.clock = clock; this.files = new ConcurrentHashMap<>(); initCurrentWorkingDirectory(); } @@ -280,7 +291,7 @@ public synchronized StoreChannel create( File fileName ) throws IOException + "' (The system cannot find the path specified)" ); } - EphemeralFileData data = new EphemeralFileData(); + EphemeralFileData data = new EphemeralFileData( clock ); free( files.put( canonicalFile( fileName ), data ) ); return new StoreFileChannel( new EphemeralFileChannel( data, new FileStillOpenException( fileName.getPath() ) ) ); @@ -369,15 +380,25 @@ public void deleteRecursively( File directory ) throws IOException } @Override - public boolean renameFile( File from, File to ) throws IOException + public boolean move( File from, File to, CopyOption... copyOptions ) throws IOException { from = canonicalFile( from ); to = canonicalFile( to ); + + boolean replaceExisting = false; + for ( CopyOption op : copyOptions ) + { + if ( op == StandardCopyOption.REPLACE_EXISTING ) + { + replaceExisting = true; + } + } + if ( !files.containsKey( from ) ) { throw new IOException( "'" + from + "' doesn't exist" ); } - if ( files.containsKey( to ) ) + if ( !replaceExisting && files.containsKey( to ) ) { throw new IOException( "'" + to + "' already exists" ); } @@ -592,6 +613,17 @@ public void truncate( File file, long size ) throws IOException data.truncate( size ); } + @Override + public long lastModifiedTime( File file ) throws IOException + { + EphemeralFileData data = files.get( canonicalFile( file ) ); + if ( data == null ) + { + throw new FileNotFoundException( "File " + file + " not found" ); + } + return data.lastModified; + } + @SuppressWarnings( "serial" ) private static class FileStillOpenException extends Exception { @@ -844,16 +876,20 @@ protected byte[] initialValue() private int size; private int forcedSize; private int locked; + private Clock clock; + private long lastModified; - public EphemeralFileData() + public EphemeralFileData( Clock clock ) { - this( new DynamicByteBuffer() ); + this( new DynamicByteBuffer(), clock ); } - private EphemeralFileData( DynamicByteBuffer data ) + private EphemeralFileData( DynamicByteBuffer data, Clock clock ) { this.fileAsBuffer = data; this.forcedBuffer = data.copy(); + this.clock = clock; + this.lastModified = clock.millis(); } int read( Positionable fc, ByteBuffer dst ) @@ -905,12 +941,13 @@ synchronized int write( Positionable fc, ByteBuffer src ) } size = newSize; + lastModified = clock.millis(); return wanted; } synchronized EphemeralFileData copy() { - EphemeralFileData copy = new EphemeralFileData( fileAsBuffer.copy() ); + EphemeralFileData copy = new EphemeralFileData( fileAsBuffer.copy(), clock ); copy.size = size; return copy; } diff --git a/community/io/src/test/java/org/neo4j/graphdb/mockfs/LimitedFilesystemAbstraction.java b/community/io/src/test/java/org/neo4j/graphdb/mockfs/LimitedFilesystemAbstraction.java index 2efb2df6f3813..054afe7f991ef 100644 --- a/community/io/src/test/java/org/neo4j/graphdb/mockfs/LimitedFilesystemAbstraction.java +++ b/community/io/src/test/java/org/neo4j/graphdb/mockfs/LimitedFilesystemAbstraction.java @@ -28,6 +28,7 @@ import java.io.Reader; import java.io.Writer; import java.nio.charset.Charset; +import java.nio.file.CopyOption; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.fs.StoreChannel; @@ -88,10 +89,10 @@ public void mkdirs( File fileName ) throws IOException } @Override - public boolean renameFile( File from, File to ) throws IOException + public boolean move( File from, File to, CopyOption... copyOptions ) throws IOException { ensureHasSpace(); - return super.renameFile( from, to ); + return super.move( from, to, copyOptions ); } public void runOutOfDiskSpace( boolean outOfSpace ) diff --git a/community/io/src/test/java/org/neo4j/graphdb/mockfs/SelectiveFileSystemAbstraction.java b/community/io/src/test/java/org/neo4j/graphdb/mockfs/SelectiveFileSystemAbstraction.java index 9eac1a5a0a82c..6c2a930ea0be1 100644 --- a/community/io/src/test/java/org/neo4j/graphdb/mockfs/SelectiveFileSystemAbstraction.java +++ b/community/io/src/test/java/org/neo4j/graphdb/mockfs/SelectiveFileSystemAbstraction.java @@ -27,6 +27,7 @@ import java.io.Reader; import java.io.Writer; import java.nio.charset.Charset; +import java.nio.file.CopyOption; import java.util.function.Function; import org.neo4j.io.fs.FileSystemAbstraction; @@ -125,9 +126,9 @@ public void deleteRecursively( File directory ) throws IOException } @Override - public boolean renameFile( File from, File to ) throws IOException + public boolean move( File from, File to, CopyOption... copyOptions ) throws IOException { - return chooseFileSystem( from ).renameFile( from, to ); + return chooseFileSystem( from ).move( from, to, copyOptions ); } @Override @@ -179,6 +180,12 @@ public void truncate( File path, long size ) throws IOException chooseFileSystem( path ).truncate( path, size ); } + @Override + public long lastModifiedTime( File file ) throws IOException + { + return chooseFileSystem( file ).lastModifiedTime( file ); + } + private FileSystemAbstraction chooseFileSystem( File file ) { return file.equals( specialFile ) ? specialFileSystem : defaultFileSystem; diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/IndexConfigStore.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/IndexConfigStore.java index 61b35206d1534..4b5e9012f578f 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/IndexConfigStore.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/IndexConfigStore.java @@ -264,7 +264,7 @@ private void write() fileSystem.deleteFile( oldFile ); try { - if ( fileSystem.fileExists( file ) && !fileSystem.renameFile( file, oldFile ) ) + if ( fileSystem.fileExists( file ) && !fileSystem.move( file, oldFile ) ) { throw new RuntimeException( "Couldn't rename " + file + " -> " + oldFile ); } @@ -277,7 +277,7 @@ private void write() // Rename the .tmp file to the current name try { - if ( !fileSystem.renameFile( tmpFile, this.file ) ) + if ( !fileSystem.move( tmpFile, this.file ) ) { throw new RuntimeException( "Couldn't rename " + tmpFile + " -> " + file ); } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/legacylogs/LegacyLogs.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/legacylogs/LegacyLogs.java index 6cf386cc319b3..65f40546015d3 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/legacylogs/LegacyLogs.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/legacylogs/LegacyLogs.java @@ -175,7 +175,7 @@ public void renameLogFiles( File storeDir ) throws IOException final String oldName = file.getName(); final long version = getLegacyLogVersion( oldName ); final String newName = DEFAULT_NAME + DEFAULT_VERSION_SUFFIX + version; - fs.renameFile( file, new File( file.getParent(), newName ) ); + fs.move( file, new File( file.getParent(), newName ) ); } deleteUnusedLogFiles( storeDir ); diff --git a/community/kernel/src/test/java/org/neo4j/test/LogTestUtils.java b/community/kernel/src/test/java/org/neo4j/test/LogTestUtils.java index 7fe4f21d67321..3fa1ff17d0d50 100644 --- a/community/kernel/src/test/java/org/neo4j/test/LogTestUtils.java +++ b/community/kernel/src/test/java/org/neo4j/test/LogTestUtils.java @@ -296,7 +296,7 @@ private static class FileBackup public void restore() throws IOException { fileSystem.deleteFile( file ); - fileSystem.renameFile( backup, file ); + fileSystem.move( backup, file ); } } diff --git a/community/logging/src/main/java/org/neo4j/logging/RotatingFileOutputStreamSupplier.java b/community/logging/src/main/java/org/neo4j/logging/RotatingFileOutputStreamSupplier.java index 83019c20af6aa..da24e1069f5cf 100644 --- a/community/logging/src/main/java/org/neo4j/logging/RotatingFileOutputStreamSupplier.java +++ b/community/logging/src/main/java/org/neo4j/logging/RotatingFileOutputStreamSupplier.java @@ -197,7 +197,7 @@ private void rotate() if ( fileSystem.fileExists( outputFile ) ) { shiftArchivedOutputFiles(); - fileSystem.renameFile( outputFile, archivedOutputFile( 1 ) ); + fileSystem.move( outputFile, archivedOutputFile( 1 ) ); } newStream = openOutputFile(); } @@ -252,7 +252,7 @@ private void shiftArchivedOutputFiles() throws IOException fileSystem.deleteFile( archive ); } else { - fileSystem.renameFile( archive, archivedOutputFile( i + 1 ) ); + fileSystem.move( archive, archivedOutputFile( i + 1 ) ); } } } diff --git a/community/logging/src/test/java/org/neo4j/logging/RotatingFileOutputStreamSupplierTest.java b/community/logging/src/test/java/org/neo4j/logging/RotatingFileOutputStreamSupplierTest.java index f4f86e669598b..925ccf85a3330 100644 --- a/community/logging/src/test/java/org/neo4j/logging/RotatingFileOutputStreamSupplierTest.java +++ b/community/logging/src/test/java/org/neo4j/logging/RotatingFileOutputStreamSupplierTest.java @@ -279,7 +279,7 @@ public void shouldNotifyListenerOnRotationErrorDuringRotationIO() throws Excepti OutputStream outputStream = supplier.get(); IOException exception = new IOException( "text exception" ); - doThrow( exception ).when( fs ).renameFile( any( File.class ), any( File.class ) ); + doThrow( exception ).when( fs ).move( any( File.class ), any( File.class ) ); write( outputStream, "A string longer than 10 bytes" ); assertThat( supplier.get(), is( outputStream ) );