From c3498d76000e4fb0cad322fd43f0f31d97937323 Mon Sep 17 00:00:00 2001 From: Ragnar Mellbin Date: Tue, 9 May 2017 12:13:30 +0200 Subject: [PATCH] Move `streamFilesRecursive` to FSA --- .../io/fs/DefaultFileSystemAbstraction.java | 7 + .../io/fs/DelegateFileSystemAbstraction.java | 6 + .../io/{pagecache => fs}/FileHandle.java | 9 +- .../neo4j/io/fs/FileSystemAbstraction.java | 28 +- .../org/neo4j/io/fs/StreamFilesRecursive.java | 94 +++ .../org/neo4j/io/fs/WrappingFileHandle.java | 91 +++ .../org/neo4j/io/pagecache/PageCache.java | 26 +- .../io/pagecache/PageSwapperFactory.java | 32 +- .../impl/SingleFilePageSwapperFactory.java | 128 +--- .../impl/muninn/MuninnPageCache.java | 64 +- .../fs/AdversarialFileSystemAbstraction.java | 9 + .../pagecache/AdversarialPageCache.java | 9 +- .../DelegatingFileSystemAbstraction.java | 9 + .../EphemeralFileSystemAbstraction.java | 9 + .../SelectiveFileSystemAbstraction.java | 10 + .../fs/DefaultFileSystemAbstractionTest.java | 2 +- .../io/fs/FileSystemAbstractionTest.java | 574 +++++++++++++++++- .../io/pagecache/DelegatingPageCache.java | 7 +- .../org/neo4j/io/pagecache/PageCacheTest.java | 525 ---------------- .../neo4j/io/pagecache/PageSwapperTest.java | 337 ---------- .../neo4j/test/rule/fs/FileSystemRule.java | 8 + .../index/labelscan/NativeLabelScanStore.java | 4 +- .../impl/storemigration/StoreUpgrader.java | 9 +- .../participant/StoreMigrator.java | 14 +- .../PageSwapperFactoryForTesting.java | 9 - .../catchup/storecopy/StoreFiles.java | 4 +- .../ReadReplicaStartupProcessTest.java | 8 +- .../storecopy/ExternallyManagedPageCache.java | 5 +- .../neo4j/com/storecopy/FileMoveAction.java | 4 +- .../org/neo4j/com/storecopy/StoreUtil.java | 10 +- .../SwitchToSlaveBranchThenCopyTest.java | 5 +- .../SwitchToSlaveCopyThenBranchTest.java | 6 +- 32 files changed, 912 insertions(+), 1150 deletions(-) rename community/io/src/main/java/org/neo4j/io/{pagecache => fs}/FileHandle.java (97%) create mode 100644 community/io/src/main/java/org/neo4j/io/fs/StreamFilesRecursive.java create mode 100644 community/io/src/main/java/org/neo4j/io/fs/WrappingFileHandle.java diff --git a/community/io/src/main/java/org/neo4j/io/fs/DefaultFileSystemAbstraction.java b/community/io/src/main/java/org/neo4j/io/fs/DefaultFileSystemAbstraction.java index 6abd591cb6572..257391a9c9066 100644 --- a/community/io/src/main/java/org/neo4j/io/fs/DefaultFileSystemAbstraction.java +++ b/community/io/src/main/java/org/neo4j/io/fs/DefaultFileSystemAbstraction.java @@ -40,6 +40,7 @@ import java.util.HashMap; import java.util.Map; import java.util.function.Function; +import java.util.stream.Stream; import org.neo4j.io.IOUtils; import org.neo4j.io.fs.watcher.DefaultFileSystemWatcher; @@ -218,6 +219,12 @@ public void deleteFileOrThrow( File file ) throws IOException Files.delete( file.toPath() ); } + @Override + public Stream streamFilesRecursive( File directory ) throws IOException + { + return StreamFilesRecursive.streamFilesRecursive( directory, this ); + } + protected StoreFileChannel getStoreFileChannel( FileChannel channel ) { return new StoreFileChannel( channel ); diff --git a/community/io/src/main/java/org/neo4j/io/fs/DelegateFileSystemAbstraction.java b/community/io/src/main/java/org/neo4j/io/fs/DelegateFileSystemAbstraction.java index d86a13cdd33e6..0be66a443c1cd 100644 --- a/community/io/src/main/java/org/neo4j/io/fs/DelegateFileSystemAbstraction.java +++ b/community/io/src/main/java/org/neo4j/io/fs/DelegateFileSystemAbstraction.java @@ -297,6 +297,12 @@ public void deleteFileOrThrow( File file ) throws IOException Files.delete( path( file ) ); } + @Override + public Stream streamFilesRecursive( File directory ) throws IOException + { + return StreamFilesRecursive.streamFilesRecursive( directory, this ); + } + @Override public void close() throws IOException { diff --git a/community/io/src/main/java/org/neo4j/io/pagecache/FileHandle.java b/community/io/src/main/java/org/neo4j/io/fs/FileHandle.java similarity index 97% rename from community/io/src/main/java/org/neo4j/io/pagecache/FileHandle.java rename to community/io/src/main/java/org/neo4j/io/fs/FileHandle.java index 7623547af69a7..7f3b3d3413f8a 100644 --- a/community/io/src/main/java/org/neo4j/io/pagecache/FileHandle.java +++ b/community/io/src/main/java/org/neo4j/io/fs/FileHandle.java @@ -17,7 +17,7 @@ * You should have received a copy of the GNU General Public License * along with this program. If not, see . */ -package org.neo4j.io.pagecache; +package org.neo4j.io.fs; import java.io.File; import java.io.IOException; @@ -25,11 +25,9 @@ import java.nio.file.CopyOption; import java.util.function.Consumer; -import org.neo4j.io.fs.FileUtils; - /** * A handle to a file as seen by the page cache. The file may or may not be mapped. - * @see PageCache#streamFilesRecursive(File) + * @see FileSystemAbstraction#streamFilesRecursive(File) */ public interface FileHandle { @@ -102,7 +100,8 @@ static Consumer handleRename( File to ) /** * Get a {@link File} object for the abstract path name that this file handle represents, and that is - * relative to the base path that was passed into the {@link PageCache#streamFilesRecursive(File)} method. + * relative to the base path that was passed into the + * {@link FileSystemAbstraction#streamFilesRecursive(File)} method. *

* This method is otherwise behaviourally the same as {@link #getFile()}. * diff --git a/community/io/src/main/java/org/neo4j/io/fs/FileSystemAbstraction.java b/community/io/src/main/java/org/neo4j/io/fs/FileSystemAbstraction.java index 12181f144d88c..3fb2c1a6b1b89 100644 --- a/community/io/src/main/java/org/neo4j/io/fs/FileSystemAbstraction.java +++ b/community/io/src/main/java/org/neo4j/io/fs/FileSystemAbstraction.java @@ -29,7 +29,9 @@ import java.io.Writer; import java.nio.charset.Charset; import java.nio.file.CopyOption; +import java.nio.file.NoSuchFileException; import java.util.function.Function; +import java.util.stream.Stream; import java.util.zip.ZipOutputStream; import org.neo4j.io.fs.watcher.FileWatcher; @@ -40,6 +42,7 @@ public interface FileSystemAbstraction extends Closeable /** * Create file watcher that provides possibilities to monitor directories on underlying file system * abstraction + * * @return specific file system abstract watcher * @throws IOException in case exception occur during file watcher creation */ @@ -83,7 +86,7 @@ public interface FileSystemAbstraction extends Closeable void copyRecursively( File fromDirectory, File toDirectory ) throws IOException; - K getOrCreateThirdPartyFileSystem( Class clazz, Function, K> creator ); + K getOrCreateThirdPartyFileSystem( Class clazz, Function,K> creator ); void truncate( File path, long size ) throws IOException; @@ -97,4 +100,27 @@ interface ThirdPartyFileSystem extends Closeable void dumpToZip( ZipOutputStream zip, byte[] scratchPad ) throws IOException; } + + /** + * Return a stream of {@link FileHandle file handles} for every file in the given directory, and its + * sub-directories. + *

+ * Alternatively, if the {@link File} given as an argument refers to a file instead of a directory, then a stream + * will be returned with a file handle for just that file. + *

+ * The stream is based on a snapshot of the file tree, so changes made to the tree using the returned file handles + * will not be reflected in the stream. + *

+ * No directories will be returned. Only files. If a file handle ends up leaving a directory empty through a + * rename or a delete, then the empty directory will automatically be deleted as well. + * Likewise, if a file is moved to a path where not all of the directories in the path exists, then those missing + * directories will be created prior to the file rename. + * + * @param directory The base directory to start streaming files from, or the specific individual file to stream. + * @return A stream of all files in the tree. + * @throws NoSuchFileException If the given base directory or file does not exists. + * @throws IOException If an I/O error occurs, possibly with the canonicalisation of the paths. + */ + Stream streamFilesRecursive( File directory ) throws IOException; + } diff --git a/community/io/src/main/java/org/neo4j/io/fs/StreamFilesRecursive.java b/community/io/src/main/java/org/neo4j/io/fs/StreamFilesRecursive.java new file mode 100644 index 0000000000000..800ad2db03f0c --- /dev/null +++ b/community/io/src/main/java/org/neo4j/io/fs/StreamFilesRecursive.java @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2002-2017 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.io.fs; + +import java.io.File; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.NoSuchFileException; +import java.util.List; +import java.util.stream.Stream; + +import static java.util.stream.Collectors.toList; + +public class StreamFilesRecursive +{ + private StreamFilesRecursive() + { + //This is a helper class, do not instantiate it. + } + + /** + * Static implementation of {@link FileSystemAbstraction#streamFilesRecursive(File)} that does not require + * any external state, other than what is presented through the given {@link FileSystemAbstraction}. + * + * Return a stream of {@link FileHandle file handles} for every file in the given directory, and its + * sub-directories. + *

+ * Alternatively, if the {@link File} given as an argument refers to a file instead of a directory, then a stream + * will be returned with a file handle for just that file. + *

+ * The stream is based on a snapshot of the file tree, so changes made to the tree using the returned file handles + * will not be reflected in the stream. + *

+ * No directories will be returned. Only files. If a file handle ends up leaving a directory empty through a + * rename or a delete, then the empty directory will automatically be deleted as well. + * Likewise, if a file is moved to a path where not all of the directories in the path exists, then those missing + * directories will be created prior to the file rename. + * + * @param directory The base directory to start streaming files from, or the specific individual file to stream. + * @param fs The {@link FileSystemAbstraction} to use for manipulating files. + * @return A {@link Stream} of {@link FileHandle}s + * @throws NoSuchFileException If the given base directory or file does not exists. + * @throws IOException If an I/O error occurs, possibly with the canonicalisation of the paths. + */ + public static Stream streamFilesRecursive( File directory, FileSystemAbstraction fs ) throws IOException + { + try + { + // We grab a snapshot of the file tree to avoid seeing the same file twice or more due to renames. + List snapshot = streamFilesRecursiveInner( directory.getCanonicalFile(), fs ).collect( toList() ); + return snapshot.stream().map( f -> new WrappingFileHandle( f, directory, fs ) ); + } + catch ( UncheckedIOException e ) + { + // We sneak checked IOExceptions through UncheckedIOExceptions due to our use of streams and lambdas. + throw e.getCause(); + } + } + + private static Stream streamFilesRecursiveInner( File directory, FileSystemAbstraction fs ) + { + File[] files = fs.listFiles( directory ); + if ( files == null ) + { + if ( !fs.fileExists( directory ) ) + { + throw new UncheckedIOException( new NoSuchFileException( directory.getPath() ) ); + } + return Stream.of( directory ); + } + else + { + return Stream.of( files ) + .flatMap( f -> fs.isDirectory( f ) ? streamFilesRecursiveInner( f, fs ) : Stream.of( f ) ); + } + } +} diff --git a/community/io/src/main/java/org/neo4j/io/fs/WrappingFileHandle.java b/community/io/src/main/java/org/neo4j/io/fs/WrappingFileHandle.java new file mode 100644 index 0000000000000..da31ef2b55729 --- /dev/null +++ b/community/io/src/main/java/org/neo4j/io/fs/WrappingFileHandle.java @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2002-2017 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.io.fs; + +import java.io.File; +import java.io.IOException; +import java.nio.file.CopyOption; + +class WrappingFileHandle implements FileHandle +{ + private final File file; + private final File baseDirectory; + private final FileSystemAbstraction fs; + + WrappingFileHandle( File file, File baseDirectory, FileSystemAbstraction fs ) + { + this.file = file; + this.baseDirectory = baseDirectory; + this.fs = fs; + } + + @Override + public File getFile() + { + return file; + } + + @Override + public File getRelativeFile() + { + int baseLength = baseDirectory.getPath().length(); + if ( baseDirectory.getParent() != null ) + { + baseLength++; + } + return new File( file.getPath().substring( baseLength ) ); + } + + @Override + public void rename( File to, CopyOption... options ) throws IOException + { + File parentFile = file.getParentFile(); + File cannonicalTarget = to.getCanonicalFile(); + fs.mkdirs( cannonicalTarget.getParentFile() ); + fs.renameFile( file, cannonicalTarget, options ); + removeEmptyParent( parentFile ); + } + + private void removeEmptyParent( File parentFile ) + { + // delete up to and including the base directory, but not above. + // Note that this may be 'null' if 'baseDirectory' is the top directory. + // Fortunately, 'File.equals(other)' handles 'null' and returns 'false' when 'other' is 'null'. + File end = baseDirectory.getParentFile(); + while ( parentFile != null && !parentFile.equals( end ) ) + { + File[] files = fs.listFiles( parentFile ); + if ( files == null || files.length > 0 ) + { + return; + } + fs.deleteFile( parentFile ); + parentFile = parentFile.getParentFile(); + } + } + + @Override + public void delete() throws IOException + { + File parentFile = file.getParentFile(); + fs.deleteFileOrThrow( file ); + removeEmptyParent( parentFile ); + } +} diff --git a/community/io/src/main/java/org/neo4j/io/pagecache/PageCache.java b/community/io/src/main/java/org/neo4j/io/pagecache/PageCache.java index d88756090f8ab..5e9ed5e4507ef 100644 --- a/community/io/src/main/java/org/neo4j/io/pagecache/PageCache.java +++ b/community/io/src/main/java/org/neo4j/io/pagecache/PageCache.java @@ -21,11 +21,11 @@ import java.io.File; import java.io.IOException; -import java.nio.file.NoSuchFileException; import java.nio.file.OpenOption; import java.nio.file.StandardOpenOption; import java.util.Optional; -import java.util.stream.Stream; + +import org.neo4j.io.fs.FileSystemAbstraction; /** * A page caching mechanism that allows caching multiple files and accessing their data @@ -110,24 +110,8 @@ public interface PageCache extends AutoCloseable int maxCachedPages(); /** - * Return a stream of {@link FileHandle file handles} for every file in the given directory, and its - * sub-directories. - *

- * Alternatively, if the {@link File} given as an argument refers to a file instead of a directory, then a stream - * will be returned with a file handle for just that file. - *

- * The stream is based on a snapshot of the file tree, so changes made to the tree using the returned file handles - * will not be reflected in the stream. - *

- * No directories will be returned. Only files. If a file handle ends up leaving a directory empty through a - * rename or a delete, then the empty directory will automatically be deleted as well. - * Likewise, if a file is moved to a path where not all of the directories in the path exists, then those missing - * directories will be created prior to the file rename. - * - * @param directory The base directory to start streaming files from, or the specific individual file to stream. - * @return A stream of all files in the tree. - * @throws NoSuchFileException If the given base directory or file does not exists. - * @throws IOException If an I/O error occurs, possibly with the canonicalisation of the paths. + * Get the {@link FileSystemAbstraction} that represents the filesystem where the paged files reside. + * @return the filesystem that the page cache is using. */ - Stream streamFilesRecursive( File directory ) throws IOException; + FileSystemAbstraction getCachedFileSystem(); } diff --git a/community/io/src/main/java/org/neo4j/io/pagecache/PageSwapperFactory.java b/community/io/src/main/java/org/neo4j/io/pagecache/PageSwapperFactory.java index de4b066cb5a31..afcb15b77f16a 100644 --- a/community/io/src/main/java/org/neo4j/io/pagecache/PageSwapperFactory.java +++ b/community/io/src/main/java/org/neo4j/io/pagecache/PageSwapperFactory.java @@ -47,6 +47,11 @@ public interface PageSwapperFactory */ void open( FileSystemAbstraction fs, Configuration config ); + /** + * Get the {@link FileSystemAbstraction} that represents the underlying storage for the page swapper. + */ + FileSystemAbstraction getFileSystemAbstraction(); + /** * Get the name of this PageSwapperFactory implementation, for configuration purpose. */ @@ -105,33 +110,6 @@ PageSwapper createPageSwapper( */ void syncDevice() throws IOException; - /** - * Return a stream of {@link FileHandle file handles} for every file in the given directory, and its - * sub-directories. - *

- * Alternatively, if the {@link File} given as an argument refers to a file instead of a directory, then a stream - * will be returned with a file handle for just that file. - *

- * The stream is based on a snapshot of the file tree, so changes made to the tree using the returned file handles - * will not be reflected in the stream. - *

- * No directories will be returned. Only files. If a file handle ends up leaving a directory empty through a - * rename or a delete, then the empty directory will automatically be deleted as well. - * Likewise, if a file is moved to a path where not all of the directories in the path exists, then those missing - * directories will be created prior to the file rename. - *

- * This method form the basis of the implementation of the {@link PageCache#streamFilesRecursive(File)} method. - * The key difference is that the stream and file handles are oblivious about which files are mapped or not. - * Thus, the returned {@link FileHandle file handles} will never throw any - * {@link org.neo4j.io.pagecache.impl.FileIsMappedException}s. - * - * @param directory The base directory to start streaming files from, or the specific individual file to stream. - * @return A stream of all files in the tree. - * @throws NoSuchFileException If the given base directory or file does not exists. - * @throws IOException If an I/O error occurs, possibly with the canonicalisation of the paths. - */ - Stream streamFilesRecursive( File directory ) throws IOException; - /** * Close and release any resources associated with this PageSwapperFactory, that it may have opened or acquired * during its construction or use. diff --git a/community/io/src/main/java/org/neo4j/io/pagecache/impl/SingleFilePageSwapperFactory.java b/community/io/src/main/java/org/neo4j/io/pagecache/impl/SingleFilePageSwapperFactory.java index 9ba02720c2bb5..d03d81d178ada 100644 --- a/community/io/src/main/java/org/neo4j/io/pagecache/impl/SingleFilePageSwapperFactory.java +++ b/community/io/src/main/java/org/neo4j/io/pagecache/impl/SingleFilePageSwapperFactory.java @@ -21,21 +21,14 @@ import java.io.File; import java.io.IOException; -import java.io.UncheckedIOException; -import java.nio.file.CopyOption; import java.nio.file.NoSuchFileException; -import java.util.List; -import java.util.stream.Stream; import org.neo4j.graphdb.config.Configuration; import org.neo4j.io.fs.FileSystemAbstraction; -import org.neo4j.io.pagecache.FileHandle; import org.neo4j.io.pagecache.PageEvictionCallback; import org.neo4j.io.pagecache.PageSwapper; import org.neo4j.io.pagecache.PageSwapperFactory; -import static java.util.stream.Collectors.toList; - /** * A factory for SingleFilePageSwapper instances. * @@ -51,6 +44,12 @@ public void open( FileSystemAbstraction fs, Configuration config ) this.fs = fs; } + @Override + public FileSystemAbstraction getFileSystemAbstraction() + { + return fs; + } + @Override public PageSwapper createPageSwapper( File file, @@ -78,62 +77,12 @@ public void syncDevice() // Nothing do to, since we `fsync` files individually in `force()`. } - @Override - public Stream streamFilesRecursive( File directory ) throws IOException - { - return streamFilesRecursive( directory.getCanonicalFile(), fs ); - } - @Override public void close() { // We have nothing to close } - /** - * Static implementation of {@link SingleFilePageSwapperFactory#streamFilesRecursive(File)} that does not require - * any external state, other than what is presented through the given {@link FileSystemAbstraction}. - * - * This method is an implementation detail of {@link PageSwapperFactory page swapper factories}, and it is made - * available here so that other {@link PageSwapperFactory} implementations can use it as the basis of their own - * implementations. In other words, so that other {@link PageSwapperFactory} implementations can implement - * {@link PageSwapperFactory#streamFilesRecursive(File)} by augmenting this stream. - * @param directory The base directory. - * @param fs The {@link FileSystemAbstraction} to use for manipulating files. - * @return A {@link Stream} of {@link FileHandle}s, as per the {@link PageSwapperFactory#streamFilesRecursive(File)} - * specification. - * @throws IOException If anything goes wrong, like {@link PageSwapperFactory#streamFilesRecursive(File)} describes. - */ - public static Stream streamFilesRecursive( File directory, FileSystemAbstraction fs ) throws IOException - { - try - { - // We grab a snapshot of the file tree to avoid seeing the same file twice or more due to renames. - List snapshot = streamFilesRecursiveInner( directory, fs ).collect( toList() ); - return snapshot.stream().map( f -> new WrappingFileHandle( f, directory, fs ) ); - } - catch ( UncheckedIOException e ) - { - // We sneak checked IOExceptions through UncheckedIOExceptions due to our use of streams and lambdas. - throw e.getCause(); - } - } - - private static Stream streamFilesRecursiveInner( File directory, FileSystemAbstraction fs ) - { - File[] files = fs.listFiles( directory ); - if ( files == null ) - { - if ( !fs.fileExists( directory ) ) - { - throw new UncheckedIOException( new NoSuchFileException( directory.getPath() ) ); - } - return Stream.of( directory ); - } - return Stream.of( files ) - .flatMap( f -> fs.isDirectory( f ) ? streamFilesRecursiveInner( f, fs ) : Stream.of( f ) ); - } - @Override public String implementationName() { @@ -158,69 +107,4 @@ public long getRequiredBufferAlignment() return 1; } - private static class WrappingFileHandle implements FileHandle - { - private final File file; - private final File baseDirectory; - private final FileSystemAbstraction fs; - - WrappingFileHandle( File file, File baseDirectory, FileSystemAbstraction fs ) - { - this.file = file; - this.baseDirectory = baseDirectory; - this.fs = fs; - } - - @Override - public File getFile() - { - return file; - } - - @Override - public File getRelativeFile() - { - int baseLength = baseDirectory.getPath().length(); - if ( baseDirectory.getParent() != null ) - { - baseLength++; - } - return new File( file.getPath().substring( baseLength ) ); - } - - @Override - public void rename( File to, CopyOption... options ) throws IOException - { - File parentFile = file.getParentFile(); - fs.mkdirs( to.getParentFile() ); - fs.renameFile( file, to, options ); - removeEmptyParent( parentFile ); - } - - private void removeEmptyParent( File parentFile ) - { - // delete up to and including the base directory, but not above. - // Note that this may be 'null' if 'baseDirectory' is the top directory. - // Fortunately, 'File.equals(other)' handles 'null' and returns 'false' when 'other' is 'null'. - File end = baseDirectory.getParentFile(); - while ( parentFile != null && !parentFile.equals( end ) ) - { - File[] files = fs.listFiles( parentFile ); - if ( files == null || files.length > 0 ) - { - return; - } - fs.deleteFile( parentFile ); - parentFile = parentFile.getParentFile(); - } - } - - @Override - public void delete() throws IOException - { - File parentFile = file.getParentFile(); - fs.deleteFileOrThrow( file ); - removeEmptyParent( parentFile ); - } - } } diff --git a/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/MuninnPageCache.java b/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/MuninnPageCache.java index c8da75bbb68aa..572e243889b92 100644 --- a/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/MuninnPageCache.java +++ b/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/MuninnPageCache.java @@ -21,7 +21,6 @@ import java.io.File; import java.io.IOException; -import java.nio.file.CopyOption; import java.nio.file.OpenOption; import java.nio.file.StandardOpenOption; import java.util.Arrays; @@ -32,9 +31,8 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.locks.LockSupport; -import java.util.stream.Stream; -import org.neo4j.io.pagecache.FileHandle; +import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.IOLimiter; import org.neo4j.io.pagecache.PageCache; import org.neo4j.io.pagecache.PageCacheOpenOptions; @@ -409,60 +407,6 @@ private MuninnPagedFile tryGetMappingOrNull( File file ) throws IOException return null; } - @Override - public Stream streamFilesRecursive( File directory ) throws IOException - { - return swapperFactory.streamFilesRecursive( directory.getCanonicalFile() ).map( this::checkingFileHandle ); - } - - private FileHandle checkingFileHandle( FileHandle fileHandle ) - { - return new FileHandle() - { - @Override - public File getFile() - { - return fileHandle.getFile(); - } - - @Override - public File getRelativeFile() - { - return fileHandle.getRelativeFile(); - } - - @Override - public void rename( File targetFile, CopyOption... options ) throws IOException - { - synchronized ( MuninnPageCache.this ) - { - File sourceFile = getFile(); - sourceFile = sourceFile.getCanonicalFile(); - targetFile = targetFile.getCanonicalFile(); - assertNotMapped( sourceFile, FileIsMappedException.Operation.RENAME ); - assertNotMapped( targetFile, FileIsMappedException.Operation.RENAME ); - fileHandle.rename( targetFile, options ); - } - } - - @Override - public String toString() - { - return fileHandle.toString(); - } - - @Override - public void delete() throws IOException - { - synchronized ( MuninnPageCache.this ) - { - assertNotMapped( getFile(), FileIsMappedException.Operation.DELETE ); - fileHandle.delete(); - } - } - }; - } - private void assertNotMapped( File file, FileIsMappedException.Operation operation ) throws IOException { if ( tryGetMappingOrNull( file ) != null ) @@ -689,6 +633,12 @@ public int maxCachedPages() return pages.length; } + @Override + public FileSystemAbstraction getCachedFileSystem() + { + return swapperFactory.getFileSystemAbstraction(); + } + int getPageCacheId() { return pageCacheId; diff --git a/community/io/src/test/java/org/neo4j/adversaries/fs/AdversarialFileSystemAbstraction.java b/community/io/src/test/java/org/neo4j/adversaries/fs/AdversarialFileSystemAbstraction.java index 967a6fe9438d5..d20fc99371b88 100644 --- a/community/io/src/test/java/org/neo4j/adversaries/fs/AdversarialFileSystemAbstraction.java +++ b/community/io/src/test/java/org/neo4j/adversaries/fs/AdversarialFileSystemAbstraction.java @@ -37,13 +37,16 @@ import java.util.HashMap; import java.util.Map; import java.util.function.Function; +import java.util.stream.Stream; import org.neo4j.adversaries.Adversary; import org.neo4j.adversaries.watcher.AdversarialFileWatcher; import org.neo4j.io.fs.DefaultFileSystemAbstraction; +import org.neo4j.io.fs.FileHandle; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.fs.StoreChannel; import org.neo4j.io.fs.StoreFileChannel; +import org.neo4j.io.fs.StreamFilesRecursive; import org.neo4j.io.fs.watcher.FileWatcher; /** @@ -230,6 +233,12 @@ public void deleteFileOrThrow( File file ) throws IOException delegate.deleteFileOrThrow( file ); } + @Override + public Stream streamFilesRecursive( File directory ) throws IOException + { + return StreamFilesRecursive.streamFilesRecursive( directory, this ); + } + private ThirdPartyFileSystem adversarialProxy( final ThirdPartyFileSystem fileSystem, Class clazz ) diff --git a/community/io/src/test/java/org/neo4j/adversaries/pagecache/AdversarialPageCache.java b/community/io/src/test/java/org/neo4j/adversaries/pagecache/AdversarialPageCache.java index f32b4ec2b8d8f..4273b448465da 100644 --- a/community/io/src/test/java/org/neo4j/adversaries/pagecache/AdversarialPageCache.java +++ b/community/io/src/test/java/org/neo4j/adversaries/pagecache/AdversarialPageCache.java @@ -24,15 +24,13 @@ import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; -import java.nio.file.NoSuchFileException; import java.nio.file.OpenOption; import java.nio.file.StandardOpenOption; import java.util.Objects; import java.util.Optional; -import java.util.stream.Stream; import org.neo4j.adversaries.Adversary; -import org.neo4j.io.pagecache.FileHandle; +import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.IOLimiter; import org.neo4j.io.pagecache.PageCache; import org.neo4j.io.pagecache.PagedFile; @@ -117,9 +115,8 @@ public int maxCachedPages() } @Override - public Stream streamFilesRecursive( File directory ) throws IOException + public FileSystemAbstraction getCachedFileSystem() { - adversary.injectFailure( NoSuchFileException.class, IOException.class ); - return delegate.streamFilesRecursive( directory ); + return delegate.getCachedFileSystem(); } } diff --git a/community/io/src/test/java/org/neo4j/graphdb/mockfs/DelegatingFileSystemAbstraction.java b/community/io/src/test/java/org/neo4j/graphdb/mockfs/DelegatingFileSystemAbstraction.java index 6d11d3bf04994..56b1e95e5eaa9 100644 --- a/community/io/src/test/java/org/neo4j/graphdb/mockfs/DelegatingFileSystemAbstraction.java +++ b/community/io/src/test/java/org/neo4j/graphdb/mockfs/DelegatingFileSystemAbstraction.java @@ -29,9 +29,12 @@ import java.nio.charset.Charset; import java.nio.file.CopyOption; import java.util.function.Function; +import java.util.stream.Stream; +import org.neo4j.io.fs.FileHandle; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.fs.StoreChannel; +import org.neo4j.io.fs.StreamFilesRecursive; import org.neo4j.io.fs.watcher.FileWatcher; public class DelegatingFileSystemAbstraction implements FileSystemAbstraction @@ -98,6 +101,12 @@ public void deleteFileOrThrow( File file ) throws IOException delegate.deleteFileOrThrow( file ); } + @Override + public Stream streamFilesRecursive( File directory ) throws IOException + { + return StreamFilesRecursive.streamFilesRecursive( directory, this ); + } + @Override public void renameFile( File from, File to, CopyOption... copyOptions ) throws IOException { diff --git a/community/io/src/test/java/org/neo4j/graphdb/mockfs/EphemeralFileSystemAbstraction.java b/community/io/src/test/java/org/neo4j/graphdb/mockfs/EphemeralFileSystemAbstraction.java index acdc963f2c4c1..b6eb5092e9bda 100644 --- a/community/io/src/test/java/org/neo4j/graphdb/mockfs/EphemeralFileSystemAbstraction.java +++ b/community/io/src/test/java/org/neo4j/graphdb/mockfs/EphemeralFileSystemAbstraction.java @@ -59,6 +59,7 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.function.Function; +import java.util.stream.Stream; import java.util.zip.CRC32; import java.util.zip.Checksum; import java.util.zip.ZipEntry; @@ -66,9 +67,11 @@ import org.neo4j.io.ByteUnit; import org.neo4j.io.IOUtils; +import org.neo4j.io.fs.FileHandle; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.fs.StoreChannel; import org.neo4j.io.fs.StoreFileChannel; +import org.neo4j.io.fs.StreamFilesRecursive; import org.neo4j.io.fs.watcher.FileWatcher; import org.neo4j.test.impl.ChannelInputStream; import org.neo4j.test.impl.ChannelOutputStream; @@ -681,6 +684,12 @@ public void deleteFileOrThrow( File file ) throws IOException } } + @Override + public Stream streamFilesRecursive( File directory ) throws IOException + { + return StreamFilesRecursive.streamFilesRecursive( directory, this ); + } + @SuppressWarnings( "serial" ) private static class FileStillOpenException extends Exception { diff --git a/community/io/src/test/java/org/neo4j/graphdb/mockfs/SelectiveFileSystemAbstraction.java b/community/io/src/test/java/org/neo4j/graphdb/mockfs/SelectiveFileSystemAbstraction.java index ba220122cb7da..6427b0d18c416 100644 --- a/community/io/src/test/java/org/neo4j/graphdb/mockfs/SelectiveFileSystemAbstraction.java +++ b/community/io/src/test/java/org/neo4j/graphdb/mockfs/SelectiveFileSystemAbstraction.java @@ -29,10 +29,13 @@ import java.nio.charset.Charset; import java.nio.file.CopyOption; import java.util.function.Function; +import java.util.stream.Stream; import org.neo4j.io.IOUtils; +import org.neo4j.io.fs.FileHandle; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.fs.StoreChannel; +import org.neo4j.io.fs.StreamFilesRecursive; import org.neo4j.io.fs.watcher.FileWatcher; /** @@ -200,6 +203,13 @@ public void deleteFileOrThrow( File file ) throws IOException chooseFileSystem( file ).deleteFileOrThrow( file ); } + @Override + public Stream streamFilesRecursive( File directory ) throws IOException + { + return StreamFilesRecursive.streamFilesRecursive( directory, this ); + + } + private FileSystemAbstraction chooseFileSystem( File file ) { return file.equals( specialFile ) ? specialFileSystem : defaultFileSystem; diff --git a/community/io/src/test/java/org/neo4j/io/fs/DefaultFileSystemAbstractionTest.java b/community/io/src/test/java/org/neo4j/io/fs/DefaultFileSystemAbstractionTest.java index 9dfc0208ad36d..2238e66f27f07 100644 --- a/community/io/src/test/java/org/neo4j/io/fs/DefaultFileSystemAbstractionTest.java +++ b/community/io/src/test/java/org/neo4j/io/fs/DefaultFileSystemAbstractionTest.java @@ -45,7 +45,7 @@ protected FileSystemAbstraction buildFileSystemAbstraction() @Test public void shouldFailGracefullyWhenPathCannotBeCreated() throws Exception { - path = new File( dir.directory(), String.valueOf( UUID.randomUUID() ) ) + path = new File( testDirectory.directory(), String.valueOf( UUID.randomUUID() ) ) { @Override public boolean mkdirs() diff --git a/community/io/src/test/java/org/neo4j/io/fs/FileSystemAbstractionTest.java b/community/io/src/test/java/org/neo4j/io/fs/FileSystemAbstractionTest.java index 6a001a5883d1d..6a26e4dece1bc 100644 --- a/community/io/src/test/java/org/neo4j/io/fs/FileSystemAbstractionTest.java +++ b/community/io/src/test/java/org/neo4j/io/fs/FileSystemAbstractionTest.java @@ -19,22 +19,39 @@ */ package org.neo4j.io.fs; +import org.hamcrest.Matchers; import org.junit.After; import org.junit.Before; import org.junit.Rule; import org.junit.Test; +import org.junit.rules.ExpectedException; import java.io.File; import java.io.IOException; import java.io.InputStream; import java.nio.ByteBuffer; +import java.nio.file.FileAlreadyExistsException; +import java.nio.file.NoSuchFileException; +import java.nio.file.StandardCopyOption; import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Set; import java.util.UUID; +import java.util.concurrent.ThreadLocalRandom; +import java.util.function.Predicate; +import java.util.stream.Stream; import org.neo4j.graphdb.mockfs.CloseTrackingFileSystem; import org.neo4j.io.fs.watcher.FileWatcher; import org.neo4j.test.rule.TestDirectory; +import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; +import static java.util.stream.Collectors.toList; +import static java.util.stream.Collectors.toSet; +import static org.apache.commons.lang3.SystemUtils.IS_OS_WINDOWS; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.nullValue; import static org.hamcrest.core.Is.is; @@ -43,12 +60,25 @@ import static org.junit.Assert.assertSame; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; +import static org.junit.Assume.assumeFalse; +import static org.junit.Assume.assumeTrue; +import static org.neo4j.io.fs.FileHandle.HANDLE_DELETE; +import static org.neo4j.io.fs.FileHandle.handleRename; +import static org.neo4j.test.matchers.ByteArrayMatcher.byteArray; public abstract class FileSystemAbstractionTest { @Rule - public TestDirectory dir = TestDirectory.testDirectory( getClass() ); - + public TestDirectory testDirectory = TestDirectory.testDirectory( getClass() ); + @Rule + public ExpectedException expectedException = ExpectedException.none(); + + private int recordSize = 9; + private int maxPages = 20; + private int pageCachePageSize = 32; + private int recordsPerFilePage = pageCachePageSize / recordSize; + private int recordCount = 25 * maxPages * recordsPerFilePage; + private int filePageSize = recordsPerFilePage * recordSize; protected FileSystemAbstraction fsa; protected File path; @@ -56,7 +86,7 @@ public abstract class FileSystemAbstractionTest public void before() throws Exception { fsa = buildFileSystemAbstraction(); - path = new File( dir.directory(), UUID.randomUUID().toString() ); + path = new File( testDirectory.directory(), UUID.randomUUID().toString() ); } @After @@ -88,7 +118,8 @@ public void shouldCreateDeepPath() throws Exception @Test public void shouldCreatePathThatAlreadyExists() throws Exception { - assertTrue( fsa.mkdir( path ) ); + fsa.mkdirs( path ); + assertTrue( fsa.fileExists( path ) ); fsa.mkdirs( path ); @@ -98,11 +129,12 @@ public void shouldCreatePathThatAlreadyExists() throws Exception @Test public void shouldCreatePathThatPointsToFile() throws Exception { - assertTrue( fsa.mkdir( path ) ); + fsa.mkdirs( path ); + assertTrue( fsa.fileExists( path ) ); path = new File( path, "some_file" ); try ( StoreChannel channel = fsa.create( path ) ) { - assertThat( channel, is( not( nullValue() )) ); + assertThat( channel, is( not( nullValue() ) ) ); fsa.mkdirs( path ); @@ -193,7 +225,7 @@ public void fileWatcherCreation() throws IOException { try ( FileWatcher fileWatcher = fsa.fileWatcher() ) { - assertNotNull( fileWatcher.watch( dir.directory( "testDirectory" ) ) ); + assertNotNull( fileWatcher.watch( testDirectory.directory( "testDirectory" ) ) ); } } @@ -216,7 +248,7 @@ public void closeThirdPartyFileSystemsOnClose() throws IOException @Test public void readAndWriteMustTakeBufferPositionIntoAccount() throws Exception { - byte[] bytes = new byte[] {1, 2, 3, 4, 5}; + byte[] bytes = new byte[]{1, 2, 3, 4, 5}; ByteBuffer buf = ByteBuffer.wrap( bytes ); buf.position( 1 ); @@ -247,4 +279,530 @@ public void readAndWriteMustTakeBufferPositionIntoAccount() throws Exception assertThat( buf.get(), is( (byte) 5 ) ); } } + + @Test + public void streamFilesRecursiveMustBeEmptyForEmptyBaseDirectory() throws Exception + { + File dir = existingDirectory( "dir" ); + assertThat( fsa.streamFilesRecursive( dir ).count(), Matchers.is( 0L ) ); + } + + @Test + public void streamFilesRecursiveMustListAllFilesInBaseDirectory() throws Exception + { + File a = existingFile( "a" ); + File b = existingFile( "b" ); + File c = existingFile( "c" ); + Stream stream = fsa.streamFilesRecursive( a.getParentFile() ); + List filepaths = stream.map( FileHandle::getFile ).collect( toList() ); + assertThat( filepaths, containsInAnyOrder( a.getCanonicalFile(), b.getCanonicalFile(), c.getCanonicalFile() ) ); + } + + @Test + public void streamFilesRecursiveMustListAllFilesInSubDirectories() throws Exception + { + File sub1 = existingDirectory( "sub1" ); + File sub2 = existingDirectory( "sub2" ); + File a = existingFile( "a" ); + File b = new File( sub1, "b" ); + File c = new File( sub2, "c" ); + ensureExists( b ); + ensureExists( c ); + + Stream stream = fsa.streamFilesRecursive( a.getParentFile() ); + List filepaths = stream.map( FileHandle::getFile ).collect( toList() ); + assertThat( filepaths, containsInAnyOrder( a.getCanonicalFile(), b.getCanonicalFile(), c.getCanonicalFile() ) ); + } + + @Test + public void streamFilesRecursiveMustNotListSubDirectories() throws Exception + { + File sub1 = existingDirectory( "sub1" ); + File sub2 = existingDirectory( "sub2" ); + File sub2sub1 = new File( sub2, "sub1" ); + ensureDirectoryExists( sub2sub1 ); + existingDirectory( "sub3" ); // must not be observed in the stream + File a = existingFile( "a" ); + File b = new File( sub1, "b" ); + File c = new File( sub2, "c" ); + ensureExists( b ); + ensureExists( c ); + + Stream stream = fsa.streamFilesRecursive( a.getParentFile() ); + List filepaths = stream.map( FileHandle::getFile ).collect( toList() ); + assertThat( filepaths, containsInAnyOrder( a.getCanonicalFile(), b.getCanonicalFile(), c.getCanonicalFile() ) ); + } + + @Test + public void streamFilesRecursiveFilePathsMustBeCanonical() throws Exception + { + File sub = existingDirectory( "sub" ); + File a = new File( new File( new File( sub, ".." ), "sub" ), "a" ); + ensureExists( a ); + + Stream stream = fsa.streamFilesRecursive( sub.getParentFile() ); + List filepaths = stream.map( FileHandle::getFile ).collect( toList() ); + assertThat( filepaths, containsInAnyOrder( a.getCanonicalFile() ) );// file in our sub directory + + } + + @Test + public void streamFilesRecursiveMustBeAbleToGivePathRelativeToBase() throws Exception + { + File sub = existingDirectory( "sub" ); + File a = existingFile( "a" ); + File b = new File( sub, "b" ); + ensureExists( b ); + File base = a.getParentFile(); + Set set = fsa.streamFilesRecursive( base ).map( FileHandle::getRelativeFile ).collect( toSet() ); + assertThat( "Files relative to base directory " + base, set, + containsInAnyOrder( new File( "a" ), new File( "sub" + File.separator + "b" ) ) ); + } + + @Test + public void streamFilesRecursiveMustListSingleFileGivenAsBase() throws Exception + { + existingDirectory( "sub" ); // must not be observed + existingFile( "sub/x" ); // must not be observed + File a = existingFile( "a" ); + + Stream stream = fsa.streamFilesRecursive( a ); + List filepaths = stream.map( FileHandle::getFile ).collect( toList() ); + assertThat( filepaths, containsInAnyOrder( a ) ); // note that we don't go into 'sub' + } + + @Test + public void streamFilesRecursiveListedSingleFileMustHaveCanonicalPath() throws Exception + { + File sub = existingDirectory( "sub" ); + existingFile( "sub/x" ); // we query specifically for 'a', so this must not be listed + File a = existingFile( "a" ); + File queryForA = new File( new File( sub, ".." ), "a" ); + + Stream stream = fsa.streamFilesRecursive( queryForA ); + List filepaths = stream.map( FileHandle::getFile ).collect( toList() ); + assertThat( filepaths, containsInAnyOrder( a.getCanonicalFile() ) ); // note that we don't go into 'sub' + } + + @Test + public void streamFilesRecursiveMustThrowOnNonExistingBasePath() throws Exception + { + File nonExisting = new File( "nonExisting" ); + expectedException.expect( NoSuchFileException.class ); + fsa.streamFilesRecursive( nonExisting ); + } + + @Test + public void streamFilesRecursiveMustRenameFiles() throws Exception + { + File a = existingFile( "a" ); + File b = nonExistingFile( "b" ); // does not yet exist + File base = a.getParentFile(); + fsa.streamFilesRecursive( base ).forEach( handleRename( b ) ); + List filepaths = fsa.streamFilesRecursive( base ).map( FileHandle::getFile ).collect( toList() ); + assertThat( filepaths, containsInAnyOrder( b.getCanonicalFile() ) ); + } + + @Test + public void streamFilesRecursiveMustDeleteFiles() throws Exception + { + File a = existingFile( "a" ); + File b = existingFile( "b" ); + File c = existingFile( "c" ); + + File base = a.getParentFile(); + fsa.streamFilesRecursive( base ).forEach( HANDLE_DELETE ); + + assertFalse( fsa.fileExists( a ) ); + assertFalse( fsa.fileExists( b ) ); + assertFalse( fsa.fileExists( c ) ); + } + + private Predicate hasFile( File a ) + { + return fh -> fh.getFile().equals( a ); + } + + @Test + public void streamFilesRecursiveMustThrowWhenDeletingNonExistingFile() throws Exception + { + File a = existingFile( "a" ); + FileHandle handle = fsa.streamFilesRecursive( a ).findAny().get(); + fsa.deleteFile( a ); + expectedException.expect( NoSuchFileException.class ); + handle.delete(); // must throw + } + + @Test + public void streamFilesRecursiveMustThrowWhenTargetFileOfRenameAlreadyExists() throws Exception + { + File a = existingFile( "a" ); + File b = existingFile( "b" ); + FileHandle handle = fsa.streamFilesRecursive( a ).findAny().get(); + expectedException.expect( FileAlreadyExistsException.class ); + handle.rename( b ); + } + + @Test + public void streamFilesRecursiveMustNotThrowWhenTargetFileOfRenameAlreadyExistsAndUsingReplaceExisting() + throws Exception + { + File a = existingFile( "a" ); + File b = existingFile( "b" ); + FileHandle handle = fsa.streamFilesRecursive( a ).findAny().get(); + handle.rename( b, StandardCopyOption.REPLACE_EXISTING ); + } + + @Test + public void streamFilesRecursiveMustDeleteSubDirectoriesEmptiedByFileRename() throws Exception + { + File sub = existingDirectory( "sub" ); + File x = new File( sub, "x" ); + ensureExists( x ); + File target = nonExistingFile( "target" ); + + fsa.streamFilesRecursive( sub ).forEach( handleRename( target ) ); + + assertFalse( fsa.isDirectory( sub ) ); + assertFalse( fsa.fileExists( sub ) ); + } + + @Test + public void streamFilesRecursiveMustDeleteMultipleLayersOfSubDirectoriesIfTheyBecomeEmptyByRename() throws Exception + { + File sub = existingDirectory( "sub" ); + File subsub = new File( sub, "subsub" ); + ensureDirectoryExists( subsub ); + File x = new File( subsub, "x" ); + ensureExists( x ); + File target = nonExistingFile( "target" ); + + fsa.streamFilesRecursive( sub ).forEach( handleRename( target ) ); + + assertFalse( fsa.isDirectory( subsub ) ); + assertFalse( fsa.fileExists( subsub ) ); + assertFalse( fsa.isDirectory( sub ) ); + assertFalse( fsa.fileExists( sub ) ); + } + + @Test + public void streamFilesRecursiveMustNotDeleteDirectoriesAboveBaseDirectoryIfTheyBecomeEmptyByRename() + throws Exception + { + File sub = existingDirectory( "sub" ); + File subsub = new File( sub, "subsub" ); + File subsubsub = new File( subsub, "subsubsub" ); + ensureDirectoryExists( subsub ); + ensureDirectoryExists( subsubsub ); + File x = new File( subsubsub, "x" ); + ensureExists( x ); + File target = nonExistingFile( "target" ); + + fsa.streamFilesRecursive( subsub ).forEach( handleRename( target ) ); + + assertFalse( fsa.fileExists( subsubsub ) ); + assertFalse( fsa.isDirectory( subsubsub ) ); + assertFalse( fsa.fileExists( subsub ) ); + assertFalse( fsa.isDirectory( subsub ) ); + assertTrue( fsa.fileExists( sub ) ); + assertTrue( fsa.isDirectory( sub ) ); + } + + @Test + public void streamFilesRecursiveMustDeleteSubDirectoriesEmptiedByFileDelete() throws Exception + { + File sub = existingDirectory( "sub" ); + File x = new File( sub, "x" ); + ensureExists( x ); + + fsa.streamFilesRecursive( sub ).forEach( HANDLE_DELETE ); + + assertFalse( fsa.isDirectory( sub ) ); + assertFalse( fsa.fileExists( sub ) ); + } + + @Test + public void streamFilesRecursiveMustDeleteMultipleLayersOfSubDirectoriesIfTheyBecomeEmptyByDelete() throws Exception + { + File sub = existingDirectory( "sub" ); + File subsub = new File( sub, "subsub" ); + ensureDirectoryExists( subsub ); + File x = new File( subsub, "x" ); + ensureExists( x ); + + fsa.streamFilesRecursive( sub ).forEach( HANDLE_DELETE ); + + assertFalse( fsa.isDirectory( subsub ) ); + assertFalse( fsa.fileExists( subsub ) ); + assertFalse( fsa.isDirectory( sub ) ); + assertFalse( fsa.fileExists( sub ) ); + } + + @Test + public void streamFilesRecursiveMustNotDeleteDirectoriesAboveBaseDirectoryIfTheyBecomeEmptyByDelete() + throws Exception + { + File sub = existingDirectory( "sub" ); + File subsub = new File( sub, "subsub" ); + File subsubsub = new File( subsub, "subsubsub" ); + ensureDirectoryExists( subsub ); + ensureDirectoryExists( subsubsub ); + File x = new File( subsubsub, "x" ); + ensureExists( x ); + + fsa.streamFilesRecursive( subsub ).forEach( HANDLE_DELETE ); + + assertFalse( fsa.fileExists( subsubsub ) ); + assertFalse( fsa.isDirectory( subsubsub ) ); + assertFalse( fsa.fileExists( subsub ) ); + assertFalse( fsa.isDirectory( subsub ) ); + assertTrue( fsa.fileExists( sub ) ); + assertTrue( fsa.isDirectory( sub ) ); + } + + @Test + public void streamFilesRecursiveMustCreateMissingPathDirectoriesImpliedByFileRename() throws Exception + { + File a = existingFile( "a" ); + File sub = new File( path, "sub" ); // does not exists + File target = new File( sub, "b" ); + + FileHandle handle = fsa.streamFilesRecursive( a ).findAny().get(); + handle.rename( target ); + + assertTrue( fsa.isDirectory( sub ) ); + assertTrue( fsa.fileExists( target ) ); + } + + @Test + public void streamFilesRecursiveMustNotSeeFilesLaterCreatedBaseDirectory() throws Exception + { + File a = existingFile( "a" ); + Stream stream = fsa.streamFilesRecursive( a.getParentFile() ); + File b = existingFile( "b" ); + Set files = stream.map( FileHandle::getFile ).collect( toSet() ); + assertThat( files, contains( a ) ); + assertThat( files, not( contains( b ) ) ); + } + + @Test + public void streamFilesRecursiveMustNotSeeFilesRenamedIntoBaseDirectory() throws Exception + { + File a = existingFile( "a" ); + File sub = existingDirectory( "sub" ); + File x = new File( sub, "x" ); + ensureExists( x ); + File target = nonExistingFile( "target" ); + Set observedFiles = new HashSet<>(); + fsa.streamFilesRecursive( a.getParentFile() ).forEach( fh -> + { + File file = fh.getFile(); + observedFiles.add( file ); + if ( file.equals( x ) ) + { + handleRename( target ).accept( fh ); + } + } ); + assertThat( observedFiles, containsInAnyOrder( a, x ) ); + } + + @Test + public void streamFilesRecursiveMustNotSeeFilesRenamedIntoSubDirectory() throws Exception + { + File a = existingFile( "a" ); + File sub = existingDirectory( "sub" ); + File target = new File( sub, "target" ); + Set observedFiles = new HashSet<>(); + fsa.streamFilesRecursive( a.getParentFile() ).forEach( fh -> + { + File file = fh.getFile(); + observedFiles.add( file ); + if ( file.equals( a ) ) + { + handleRename( target ).accept( fh ); + } + } ); + assertThat( observedFiles, containsInAnyOrder( a ) ); + } + + @Test + public void streamFilesRecursiveRenameMustCanonicaliseSourceFile() throws Exception + { + // File 'a' should canonicalise from 'a/poke/..' to 'a', which is a file that exists. + // Thus, this should not throw a NoSuchFileException. + File a = new File( new File( existingFile( "a" ), "poke" ), ".." ); + File b = nonExistingFile( "b" ); + + FileHandle handle = fsa.streamFilesRecursive( a ).findAny().get(); + handle.rename( b ); // must not throw + } + + @Test + public void streamFilesRecursiveRenameMustCanonicaliseTargetFile() throws Exception + { + // File 'b' should canonicalise from 'b/poke/..' to 'b', which is a file that doesn't exists. + // Thus, this should not throw a NoSuchFileException for the 'poke' directory. + File a = existingFile( "a" ); + File b = new File( new File( new File( path, "b" ), "poke" ), ".." ); + FileHandle handle = fsa.streamFilesRecursive( a ).findAny().get(); + handle.rename( b ); + } + + @Test + public void streamFilesRecursiveRenameTargetFileMustBeRenamed() throws Exception + { + File a = existingFile( "a" ); + File b = nonExistingFile( "b" ); + FileHandle handle = fsa.streamFilesRecursive( a ).findAny().get(); + handle.rename( b ); + assertTrue( fsa.fileExists( b ) ); + } + + @Test + public void streamFilesRecursiveSourceFileMustNotBeMappableAfterRename() throws Exception + { + File a = existingFile( "a" ); + File b = nonExistingFile( "b" ); + FileHandle handle = fsa.streamFilesRecursive( a ).findAny().get(); + handle.rename( b ); + assertFalse( fsa.fileExists( a ) ); + + } + + @Test + public void streamFilesRecursiveRenameMustNotChangeSourceFileContents() throws Exception + { + File a = existingFile( "a" ); + File b = nonExistingFile( "b" ); + generateFileWithRecords( a, recordCount ); + FileHandle handle = fsa.streamFilesRecursive( a ).findAny().get(); + handle.rename( b ); + verifyRecordsInFile( b, recordCount ); + } + + @Test + public void streamFilesRecursiveRenameMustNotChangeSourceFileContentsWithReplaceExisting() throws Exception + { + File a = existingFile( "a" ); + File b = existingFile( "b" ); + generateFileWithRecords( a, recordCount ); + generateFileWithRecords( b, recordCount + recordsPerFilePage ); + + // Fill 'b' with random data + try ( StoreChannel channel = fsa.open( b, "rw" ) ) + { + ThreadLocalRandom rng = ThreadLocalRandom.current(); + int fileSize = (int) channel.size(); + ByteBuffer buffer = ByteBuffer.allocate( fileSize ); + for ( int i = 0; i < fileSize; i++ ) + + { + buffer.put( i, (byte) rng.nextInt() ); + } + buffer.rewind(); + channel.writeAll( buffer ); + } + + // Do the rename + FileHandle handle = fsa.streamFilesRecursive( a ).findAny().get(); + handle.rename( b, REPLACE_EXISTING ); + + // Then verify that the old random data we put in 'b' has been replaced with the contents of 'a' + verifyRecordsInFile( b, recordCount ); + + } + + private void generateFileWithRecords( File file, int recordCount ) throws IOException + { + try ( StoreChannel channel = fsa.open( file, "rw" ) ) + { + ByteBuffer buf = ByteBuffer.allocate( recordSize ); + for ( int i = 0; i < recordCount; i++ ) + { + generateRecordForId( i, buf ); + int rem = buf.remaining(); + do + { + rem -= channel.write( buf ); + } + while ( rem > 0 ); + } + } + } + + private void verifyRecordsInFile( File file, int recordCount ) throws IOException + { + try ( StoreChannel channel = fsa.open( file, "r" ) ) + { + ByteBuffer buf = ByteBuffer.allocate( recordSize ); + ByteBuffer observation = ByteBuffer.allocate( recordSize ); + for ( int i = 0; i < recordCount; i++ ) + { + generateRecordForId( i, buf ); + observation.position( 0 ); + channel.read( observation ); + assertRecord( i, observation, buf ); + } + } + } + + private void assertRecord( long pageId, ByteBuffer actualPageContents, ByteBuffer expectedPageContents ) + { + byte[] actualBytes = actualPageContents.array(); + byte[] expectedBytes = expectedPageContents.array(); + int estimatedPageId = estimateId( actualBytes ); + assertThat( "Page id: " + pageId + " " + "(based on record data, it should have been " + estimatedPageId + + ", a difference of " + Math.abs( pageId - estimatedPageId ) + ")", actualBytes, + byteArray( expectedBytes ) ); + } + + private int estimateId( byte[] record ) + { + return ByteBuffer.wrap( record ).getInt() - 1; + } + + private static void generateRecordForId( long id, ByteBuffer buf ) + { + buf.position( 0 ); + int x = (int) (id + 1); + buf.putInt( x ); + while ( buf.position() < buf.limit() ) + { + x++; + buf.put( (byte) (x & 0xFF) ); + } + buf.position( 0 ); + } + + private File existingFile( String fileName ) throws IOException + { + File file = new File( path, fileName ); + fsa.mkdirs( path ); + fsa.create( file ).close(); + return file; + } + + private File nonExistingFile( String fileName ) throws IOException + { + File file = new File( path, fileName ); + return file; + } + + private File existingDirectory( String dir ) throws IOException + { + File directory = new File( path, dir ); + fsa.mkdirs( directory ); + return directory; + } + + private void ensureExists( File file ) throws IOException + { + fsa.mkdirs( file.getParentFile() ); + fsa.create( file ).close(); + } + + private void ensureDirectoryExists( File directory ) throws IOException + { + fsa.mkdirs( directory ); + } } diff --git a/community/io/src/test/java/org/neo4j/io/pagecache/DelegatingPageCache.java b/community/io/src/test/java/org/neo4j/io/pagecache/DelegatingPageCache.java index f5772f1cc7052..f37470ce1f99b 100644 --- a/community/io/src/test/java/org/neo4j/io/pagecache/DelegatingPageCache.java +++ b/community/io/src/test/java/org/neo4j/io/pagecache/DelegatingPageCache.java @@ -25,6 +25,9 @@ import java.util.Optional; import java.util.stream.Stream; +import org.neo4j.io.fs.FileHandle; +import org.neo4j.io.fs.FileSystemAbstraction; + public class DelegatingPageCache implements PageCache { private final PageCache delegate; @@ -61,9 +64,9 @@ public int maxCachedPages() } @Override - public Stream streamFilesRecursive( File directory ) throws IOException + public FileSystemAbstraction getCachedFileSystem() { - return delegate.streamFilesRecursive( directory ); + return delegate.getCachedFileSystem(); } public void flushAndForce( IOLimiter limiter ) throws IOException diff --git a/community/io/src/test/java/org/neo4j/io/pagecache/PageCacheTest.java b/community/io/src/test/java/org/neo4j/io/pagecache/PageCacheTest.java index fecc51dc862b7..087dcb2b6eacc 100644 --- a/community/io/src/test/java/org/neo4j/io/pagecache/PageCacheTest.java +++ b/community/io/src/test/java/org/neo4j/io/pagecache/PageCacheTest.java @@ -34,17 +34,13 @@ import java.nio.channels.ClosedChannelException; import java.nio.channels.ReadableByteChannel; import java.nio.channels.WritableByteChannel; -import java.nio.file.FileAlreadyExistsException; import java.nio.file.NoSuchFileException; import java.nio.file.OpenOption; -import java.nio.file.StandardCopyOption; import java.nio.file.StandardOpenOption; -import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Optional; import java.util.Queue; -import java.util.Set; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.CountDownLatch; import java.util.concurrent.Future; @@ -52,8 +48,6 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; -import java.util.function.Predicate; -import java.util.stream.Stream; import org.neo4j.adversaries.RandomAdversary; import org.neo4j.adversaries.pagecache.AdversarialPagedFile; @@ -65,7 +59,6 @@ import org.neo4j.graphdb.mockfs.EphemeralFileSystemAbstraction; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.fs.StoreChannel; -import org.neo4j.io.pagecache.impl.FileIsMappedException; import org.neo4j.io.pagecache.impl.SingleFilePageSwapperFactory; import org.neo4j.io.pagecache.randomharness.Record; import org.neo4j.io.pagecache.randomharness.StandardRecordFormat; @@ -80,21 +73,14 @@ import static java.lang.Long.toHexString; import static java.lang.System.currentTimeMillis; -import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; import static java.nio.file.StandardOpenOption.DELETE_ON_CLOSE; -import static java.util.stream.Collectors.toList; -import static java.util.stream.Collectors.toSet; import static org.hamcrest.Matchers.both; -import static org.hamcrest.Matchers.contains; -import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.isA; import static org.hamcrest.Matchers.lessThan; import static org.hamcrest.Matchers.lessThanOrEqualTo; -import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.nullValue; import static org.hamcrest.Matchers.sameInstance; import static org.junit.Assert.assertArrayEquals; @@ -104,8 +90,6 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import static org.junit.Assume.assumeTrue; -import static org.neo4j.io.pagecache.FileHandle.HANDLE_DELETE; -import static org.neo4j.io.pagecache.FileHandle.handleRename; import static org.neo4j.io.pagecache.PagedFile.PF_NO_FAULT; import static org.neo4j.io.pagecache.PagedFile.PF_NO_GROW; import static org.neo4j.io.pagecache.PagedFile.PF_SHARED_READ_LOCK; @@ -5171,515 +5155,6 @@ public void fileSizeMustIncreaseInPageIncrements() throws Exception } } - @Test - public void streamFilesRecursiveMustBeEmptyForEmptyBaseDirectory() throws Exception - { - configureStandardPageCache(); - File dir = existingDirectory( "dir" ); - assertThat( pageCache.streamFilesRecursive( dir ).count(), is( 0L ) ); - } - - @Test - public void streamFilesRecursiveMustListAllFilesInBaseDirectory() throws Exception - { - configureStandardPageCache(); - File a = existingFile( "a" ); - File b = existingFile( "b" ); - File c = existingFile( "c" ); - Stream stream = pageCache.streamFilesRecursive( a.getParentFile() ); - List filepaths = stream.map( FileHandle::getFile ).collect( toList() ); - assertThat( filepaths, containsInAnyOrder( a.getCanonicalFile(), b.getCanonicalFile(), c.getCanonicalFile() ) ); - } - - @Test - public void streamFilesRecursiveMustListAllFilesInSubDirectories() throws Exception - { - configureStandardPageCache(); - File sub1 = existingDirectory( "sub1" ); - File sub2 = existingDirectory( "sub2" ); - File a = existingFile( "a" ); - File b = new File( sub1, "b" ); - File c = new File( sub2, "c" ); - ensureExists( b ); - ensureExists( c ); - - Stream stream = pageCache.streamFilesRecursive( a.getParentFile() ); - List filepaths = stream.map( FileHandle::getFile ).collect( toList() ); - assertThat( filepaths, containsInAnyOrder( a.getCanonicalFile(), b.getCanonicalFile(), c.getCanonicalFile() ) ); - } - - @Test - public void streamFilesRecursiveMustNotListSubDirectories() throws Exception - { - configureStandardPageCache(); - File sub1 = existingDirectory( "sub1" ); - File sub2 = existingDirectory( "sub2" ); - File sub2sub1 = new File( sub2, "sub1" ); - ensureDirectoryExists( sub2sub1 ); - existingDirectory( "sub3" ); // must not be observed in the stream - File a = existingFile( "a" ); - File b = new File( sub1, "b" ); - File c = new File( sub2, "c" ); - ensureExists( b ); - ensureExists( c ); - - Stream stream = pageCache.streamFilesRecursive( a.getParentFile() ); - List filepaths = stream.map( FileHandle::getFile ).collect( toList() ); - assertThat( filepaths, containsInAnyOrder( a.getCanonicalFile(), b.getCanonicalFile(), c.getCanonicalFile() ) ); - } - - @Test - public void streamFilesRecursiveFilePathsMustBeCanonical() throws Exception - { - configureStandardPageCache(); - File sub = existingDirectory( "sub" ); - File a = new File( new File( new File( sub, ".." ), "sub" ), "a" ); - ensureExists( a ); - - Stream stream = pageCache.streamFilesRecursive( sub.getParentFile() ); - List filepaths = stream.map( FileHandle::getFile ).collect( toList() ); - assertThat( filepaths, containsInAnyOrder( - a.getCanonicalFile(), // file in our sub directory - file( "a" ).getCanonicalFile() ) ); // this file is always created by the test setup - } - - @Test - public void streamFilesRecursiveMustBeAbleToGivePathRelativeToBase() throws Exception - { - configureStandardPageCache(); - File sub = existingDirectory( "sub" ); - File a = file( "a" ); - File b = new File( sub, "b" ); - ensureExists( b ); - File base = a.getParentFile(); - Set set = pageCache.streamFilesRecursive( base ).map( FileHandle::getRelativeFile ).collect( toSet() ); - assertThat( "Files relative to base directory " + base, - set, containsInAnyOrder( new File( "a" ), new File( "sub" + File.separator + "b" ) ) ); - } - - @Test - public void streamFilesRecursiveMustListSingleFileGivenAsBase() throws Exception - { - configureStandardPageCache(); - existingDirectory( "sub" ); // must not be observed - existingFile( "sub/x" ); // must not be observed - File a = file( "a" ); - - Stream stream = pageCache.streamFilesRecursive( a ); - List filepaths = stream.map( FileHandle::getFile ).collect( toList() ); - assertThat( filepaths, containsInAnyOrder( a.getCanonicalFile() ) ); // note that we don't go into 'sub' - } - - @Test - public void streamFilesRecursiveListedSingleFileMustHaveCanonicalPath() throws Exception - { - configureStandardPageCache(); - File sub = existingDirectory( "sub" ); - existingFile( "sub/x" ); // we query specifically for 'a', so this must not be listed - File a = file( "a" ); - File queryForA = new File( new File( sub, ".." ), "a" ); - - Stream stream = pageCache.streamFilesRecursive( queryForA ); - List filepaths = stream.map( FileHandle::getFile ).collect( toList() ); - assertThat( filepaths, containsInAnyOrder( a.getCanonicalFile() ) ); // note that we don't go into 'sub' - } - - @Test - public void streamFilesRecursiveMustThrowOnNonExistingBasePath() throws Exception - { - configureStandardPageCache(); - File nonExisting = file( "nonExisting" ); - expectedException.expect( NoSuchFileException.class ); - pageCache.streamFilesRecursive( nonExisting ); - } - - @Test - public void streamFilesRecursiveMustRenameFiles() throws Exception - { - configureStandardPageCache(); - File a = file( "a" ); - File b = file( "b" ); // does not yet exist - File base = a.getParentFile(); - pageCache.streamFilesRecursive( base ).forEach( handleRename( b ) ); - List filepaths = pageCache.streamFilesRecursive( base ).map( FileHandle::getFile ).collect( toList() ); - assertThat( filepaths, containsInAnyOrder( b.getCanonicalFile() ) ); - } - - @Test - public void streamFilesRecursiveMustDeleteFiles() throws Exception - { - configureStandardPageCache(); - File a = file( "a" ); - File b = file( "b" ); - File c = file( "c" ); - ensureExists( a ); - ensureExists( b ); - ensureExists( c ); - - File base = a.getParentFile(); - pageCache.streamFilesRecursive( base ).forEach( HANDLE_DELETE ); - - assertFalse( fs.fileExists( a ) ); - assertFalse( fs.fileExists( b ) ); - assertFalse( fs.fileExists( c ) ); - } - - @Test - public void streamFilesRecursiveMustThrowWhenRenamingMappedSourceFile() throws Exception - { - configureStandardPageCache(); - File a = file( "a" ); - File b = file( "b" ); - try ( PagedFile ignore = pageCache.map( a, filePageSize ) ) - { - pageCache.streamFilesRecursive( a.getParentFile() ).forEach( fh -> - { - expectedException.expectCause( isA( FileIsMappedException.class ) ); - handleRename( b ).accept( fh ); - } ); - } - } - - @Test - public void streamFilesRecursiveMustThrowWhenRenamingMappedTargetFile() throws Exception - { - configureStandardPageCache(); - File a = file( "a" ); - File b = existingFile( "b" ); - try ( PagedFile ignore = pageCache.map( b, filePageSize ) ) - { - pageCache.streamFilesRecursive( a.getParentFile() ).filter( hasFile( a ) ).forEach( fh -> - { - expectedException.expectCause( isA( FileIsMappedException.class ) ); - handleRename( b ).accept( fh ); - } ); - } - } - - private Predicate hasFile( File a ) - { - return fh -> fh.getFile().equals( a ); - } - - @Test - public void streamFilesRecursiveMustThrowWhenDeletingMappedFile() throws Exception - { - configureStandardPageCache(); - File a = file( "a" ); - try ( PagedFile ignore = pageCache.map( a, filePageSize ) ) - { - FileHandle handle = pageCache.streamFilesRecursive( a ).findAny().get(); - expectedException.expect( FileIsMappedException.class ); - handle.delete(); - } - } - - @Test - public void streamFilesRecursiveMustThrowWhenDeletingNonExistingFile() throws Exception - { - configureStandardPageCache(); - File a = file( "a" ); - FileHandle handle = pageCache.streamFilesRecursive( a ).findAny().get(); - fs.deleteFile( a ); - expectedException.expect( NoSuchFileException.class ); - handle.delete(); // must throw - } - - @Test - public void streamFilesRecursiveMustThrowWhenTargetFileOfRenameAlreadyExists() throws Exception - { - configureStandardPageCache(); - File a = file( "a" ); - File b = existingFile( "b" ); - FileHandle handle = pageCache.streamFilesRecursive( a ).findAny().get(); - expectedException.expect( FileAlreadyExistsException.class ); - handle.rename( b ); - } - - @Test - public void streamFilesRecursiveMustNotThrowWhenTargetFileOfRenameAlreadyExistsAndUsingReplaceExisting() - throws Exception - { - configureStandardPageCache(); - File a = file( "a" ); - File b = existingFile( "b" ); - FileHandle handle = pageCache.streamFilesRecursive( a ).findAny().get(); - handle.rename( b, StandardCopyOption.REPLACE_EXISTING ); - } - - @Test - public void streamFilesRecursiveMustDeleteSubDirectoriesEmptiedByFileRename() throws Exception - { - configureStandardPageCache(); - File sub = existingDirectory( "sub" ); - File x = new File( sub, "x" ); - ensureExists( x ); - File target = file( "target" ); - - pageCache.streamFilesRecursive( sub ).forEach( handleRename( target ) ); - - assertFalse( fs.isDirectory( sub ) ); - assertFalse( fs.fileExists( sub ) ); - } - - @Test - public void streamFilesRecursiveMustDeleteMultipleLayersOfSubDirectoriesIfTheyBecomeEmptyByRename() throws Exception - { - configureStandardPageCache(); - File sub = existingDirectory( "sub" ); - File subsub = new File( sub, "subsub" ); - ensureDirectoryExists( subsub ); - File x = new File( subsub, "x" ); - ensureExists( x ); - File target = file( "target" ); - - pageCache.streamFilesRecursive( sub ).forEach( handleRename( target ) ); - - assertFalse( fs.isDirectory( subsub ) ); - assertFalse( fs.fileExists( subsub ) ); - assertFalse( fs.isDirectory( sub ) ); - assertFalse( fs.fileExists( sub ) ); - } - - @Test - public void streamFilesRecursiveMustNotDeleteDirectoriesAboveBaseDirectoryIfTheyBecomeEmptyByRename() throws Exception - { - configureStandardPageCache(); - File sub = existingDirectory( "sub" ); - File subsub = new File( sub, "subsub" ); - File subsubsub = new File( subsub, "subsubsub" ); - ensureDirectoryExists( subsub ); - ensureDirectoryExists( subsubsub ); - File x = new File( subsubsub, "x" ); - ensureExists( x ); - File target = file( "target" ); - - pageCache.streamFilesRecursive( subsub ).forEach( handleRename( target ) ); - - assertFalse( fs.fileExists( subsubsub ) ); - assertFalse( fs.isDirectory( subsubsub ) ); - assertFalse( fs.fileExists( subsub ) ); - assertFalse( fs.isDirectory( subsub ) ); - assertTrue( fs.fileExists( sub ) ); - assertTrue( fs.isDirectory( sub ) ); - } - - @Test - public void streamFilesRecursiveMustDeleteSubDirectoriesEmptiedByFileDelete() throws Exception - { - configureStandardPageCache(); - File sub = existingDirectory( "sub" ); - File x = new File( sub, "x" ); - ensureExists( x ); - - pageCache.streamFilesRecursive( sub ).forEach( HANDLE_DELETE ); - - assertFalse( fs.isDirectory( sub ) ); - assertFalse( fs.fileExists( sub ) ); - } - - @Test - public void streamFilesRecursiveMustDeleteMultipleLayersOfSubDirectoriesIfTheyBecomeEmptyByDelete() throws Exception - { - configureStandardPageCache(); - File sub = existingDirectory( "sub" ); - File subsub = new File( sub, "subsub" ); - ensureDirectoryExists( subsub ); - File x = new File( subsub, "x" ); - ensureExists( x ); - - pageCache.streamFilesRecursive( sub ).forEach( HANDLE_DELETE ); - - assertFalse( fs.isDirectory( subsub ) ); - assertFalse( fs.fileExists( subsub ) ); - assertFalse( fs.isDirectory( sub ) ); - assertFalse( fs.fileExists( sub ) ); - } - - @Test - public void streamFilesRecursiveMustNotDeleteDirectoriesAboveBaseDirectoryIfTheyBecomeEmptyByDelete() throws Exception - { - configureStandardPageCache(); - File sub = existingDirectory( "sub" ); - File subsub = new File( sub, "subsub" ); - File subsubsub = new File( subsub, "subsubsub" ); - ensureDirectoryExists( subsub ); - ensureDirectoryExists( subsubsub ); - File x = new File( subsubsub, "x" ); - ensureExists( x ); - - pageCache.streamFilesRecursive( subsub ).forEach( HANDLE_DELETE ); - - assertFalse( fs.fileExists( subsubsub ) ); - assertFalse( fs.isDirectory( subsubsub ) ); - assertFalse( fs.fileExists( subsub ) ); - assertFalse( fs.isDirectory( subsub ) ); - assertTrue( fs.fileExists( sub ) ); - assertTrue( fs.isDirectory( sub ) ); - } - - @Test - public void streamFilesRecursiveMustCreateMissingPathDirectoriesImpliedByFileRename() throws Exception - { - configureStandardPageCache(); - File a = file( "a" ); - File sub = file( "sub" ); // does not exists - File target = new File( sub, "b" ); - - FileHandle handle = pageCache.streamFilesRecursive( a ).findAny().get(); - handle.rename( target ); - - assertTrue( fs.isDirectory( sub ) ); - assertTrue( fs.fileExists( target ) ); - } - - @Test - public void streamFilesRecursiveMustNotSeeFilesLaterCreatedBaseDirectory() throws Exception - { - configureStandardPageCache(); - File a = file( "a" ); - Stream stream = pageCache.streamFilesRecursive( a.getParentFile() ); - File b = existingFile( "b" ); - Set files = stream.map( FileHandle::getFile ).collect( toSet() ); - assertThat( files, contains( a ) ); - assertThat( files, not( contains( b ) ) ); - } - - @Test - public void streamFilesRecursiveMustNotSeeFilesRenamedIntoBaseDirectory() throws Exception - { - configureStandardPageCache(); - File a = file( "a" ); - File sub = existingDirectory( "sub" ); - File x = new File( sub, "x" ); - ensureExists( x ); - File target = file( "target" ); - Set observedFiles = new HashSet<>(); - pageCache.streamFilesRecursive( a.getParentFile() ).forEach( fh -> - { - File file = fh.getFile(); - observedFiles.add( file ); - if ( file.equals( x ) ) - { - handleRename( target ).accept( fh ); - } - } ); - assertThat( observedFiles, containsInAnyOrder( a, x ) ); - } - - @Test - public void streamFilesRecursiveMustNotSeeFilesRenamedIntoSubDirectory() throws Exception - { - configureStandardPageCache(); - File a = file( "a" ); - File sub = existingDirectory( "sub" ); - File target = new File( sub, "target" ); - Set observedFiles = new HashSet<>(); - pageCache.streamFilesRecursive( a.getParentFile() ).forEach( fh -> - { - File file = fh.getFile(); - observedFiles.add( file ); - if ( file.equals( a ) ) - { - handleRename( target ).accept( fh ); - } - } ); - assertThat( observedFiles, containsInAnyOrder( a ) ); - } - - @Test - public void streamFilesRecursiveRenameMustCanonicaliseSourceFile() throws Exception - { - configureStandardPageCache(); - // File 'a' should canonicalise from 'a/poke/..' to 'a', which is a file that exists. - // Thus, this should not throw a NoSuchFileException. - File a = new File( new File( file( "a" ), "poke" ), ".." ); - File b = file( "b" ); - - FileHandle handle = pageCache.streamFilesRecursive( a ).findAny().get(); - handle.rename( b ); // must not throw - } - - @Test - public void streamFilesRecursiveRenameMustCanonicaliseTargetFile() throws Exception - { - configureStandardPageCache(); - // File 'b' should canonicalise from 'b/poke/..' to 'b', which is a file that doesn't exists. - // Thus, this should not throw a NoSuchFileException for the 'poke' directory. - File a = file( "a" ); - File b = new File( new File( file( "b" ), "poke" ), ".." ); - FileHandle handle = pageCache.streamFilesRecursive( a ).findAny().get(); - handle.rename( b ); - } - - @Test - public void streamFilesRecursiveRenameTargetFileMustBeMappable() throws Exception - { - configureStandardPageCache(); - File a = file( "a" ); - File b = file( "b" ); - FileHandle handle = pageCache.streamFilesRecursive( a ).findAny().get(); - handle.rename( b ); - pageCache.map( b, filePageSize ).close(); - } - - @Test - public void streamFilesRecursiveSourceFileMustNotBeMappableAfterRename() throws Exception - { - configureStandardPageCache(); - File a = file( "a" ); - File b = file( "b" ); - FileHandle handle = pageCache.streamFilesRecursive( a ).findAny().get(); - handle.rename( b ); - expectedException.expect( NoSuchFileException.class ); - pageCache.map( a, filePageSize ); - fail( "pageCache.map should have thrown" ); - } - - @Test - public void streamFilesRecursiveRenameMustNotChangeSourceFileContents() throws Exception - { - configureStandardPageCache(); - File a = file( "a" ); - File b = file( "b" ); - generateFileWithRecords( a, recordCount, recordSize ); - FileHandle handle = pageCache.streamFilesRecursive( a ).findAny().get(); - handle.rename( b ); - verifyRecordsInFile( b, recordCount ); - } - - @Test - public void streamFilesRecursiveRenameMustNotChangeSourceFileContentsWithReplaceExisting() throws Exception - { - configureStandardPageCache(); - File a = file( "a" ); - File b = existingFile( "b" ); - generateFileWithRecords( a, recordCount, recordSize ); - generateFileWithRecords( b, recordCount + recordsPerFilePage, recordSize ); - - // Fill 'b' with random data - try ( PagedFile pf = pageCache.map( b, filePageSize ); - PageCursor cursor = pf.io( 0, PF_SHARED_WRITE_LOCK | PF_NO_GROW ) ) - { - ThreadLocalRandom rng = ThreadLocalRandom.current(); - while ( cursor.next() ) - { - int pageSize = cursor.getCurrentPageSize(); - for ( int i = 0; i < pageSize; i++ ) - { - cursor.putByte( i, (byte) rng.nextInt() ); - } - } - } - - // Do the rename - FileHandle handle = pageCache.streamFilesRecursive( a ).findAny().get(); - handle.rename( b, REPLACE_EXISTING ); - - // Then verify that the old random data we put in 'b' has been replaced with the contents of 'a' - verifyRecordsInFile( b, recordCount ); - } - @Test public void shouldZeroAllBytesOnClear() throws Exception { diff --git a/community/io/src/test/java/org/neo4j/io/pagecache/PageSwapperTest.java b/community/io/src/test/java/org/neo4j/io/pagecache/PageSwapperTest.java index dff9d0c24ed39..32fc015a234a6 100644 --- a/community/io/src/test/java/org/neo4j/io/pagecache/PageSwapperTest.java +++ b/community/io/src/test/java/org/neo4j/io/pagecache/PageSwapperTest.java @@ -30,12 +30,9 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.nio.channels.ClosedChannelException; -import java.nio.file.FileAlreadyExistsException; import java.nio.file.NoSuchFileException; import java.util.ArrayList; -import java.util.HashSet; import java.util.List; -import java.util.Set; import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.CountDownLatch; @@ -46,15 +43,10 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; -import java.util.stream.Stream; import org.neo4j.io.pagecache.impl.ByteBufferPage; import org.neo4j.test.rule.TestDirectory; -import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; -import static java.util.stream.Collectors.toSet; -import static org.apache.commons.lang3.SystemUtils.IS_OS_WINDOWS; -import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.isOneOf; import static org.hamcrest.Matchers.sameInstance; @@ -62,8 +54,6 @@ import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; -import static org.junit.Assume.assumeFalse; -import static org.junit.Assume.assumeTrue; @SuppressWarnings( "OptionalGetWithoutIsPresent" ) public abstract class PageSwapperTest @@ -1288,331 +1278,4 @@ public void mustDeleteFileIfClosedWithCloseAndDelete() throws Exception // Just as planned! } } - - @Test - public void streamFilesRecursiveMustBeEmptyForEmptyBaseDirectory() throws Exception - { - PageSwapperFactory factory = createSwapperFactory(); - assertThat( factory.streamFilesRecursive( baseDirectory() ).count(), is( 0L ) ); - } - - @Test - public void streamFilesRecursiveMustListAllFilesInBaseDirectory() throws Exception - { - PageSwapperFactory factory = createSwapperFactory(); - File base = baseDirectory(); - File a = new File( base, "a" ); - File b = new File( base, "b" ); - createSwapperAndFile( factory, a ); - createSwapperAndFile( factory, b ); - Set files = factory.streamFilesRecursive( base ).map( FileHandle::getFile ).collect( toSet() ); - assertThat( files, containsInAnyOrder( a, b ) ); - } - - @Test - public void streamFilesRecursiveMustListAllFilesInSubDirectories() throws Exception - { - PageSwapperFactory factory = createSwapperFactory(); - File base = baseDirectory(); - File sub1 = new File( base, "sub1" ); - File sub1sub1 = new File( sub1, "sub1" ); - File sub2 = new File( base, "sub2" ); - File sub3 = new File( base, "sub3" ); - mkdirs( sub1 ); - mkdirs( sub1sub1 ); - mkdirs( sub2 ); - mkdirs( sub3 ); // empty, not listed - File a = new File( base, "a" ); - File b = new File( sub1, "b" ); - File c = new File( sub1sub1, "c" ); - File d = new File( sub1sub1, "d" ); - File e = new File( sub2, "e" ); - File[] files = new File[] {a, b, c, d, e}; - for ( File f : files ) - { - createSwapperAndFile( factory, f ); - } - Set set = factory.streamFilesRecursive( base ).map( FileHandle::getFile ).collect( toSet() ); - assertThat( set, containsInAnyOrder( files ) ); - } - - @Test - public void streamFilesRecursiveFilePathsMustBeCanonical() throws Exception - { - PageSwapperFactory factory = createSwapperFactory(); - File base = baseDirectory(); - File sub = new File( base, "sub" ); - mkdirs( sub ); - File a = new File( new File( new File( sub, ".." ), "sub" ), "a" ); - File canonicalFile = a.getCanonicalFile(); - createSwapperAndFile( factory, canonicalFile ); - String actualPath = factory.streamFilesRecursive( a ) - .map( fh -> fh.getFile().getAbsolutePath() ).findAny().get(); - assertThat( actualPath, is( canonicalFile.getAbsolutePath() ) ); - } - - @Test - public void streamFilesRecursiveMustBeAbleToGivePathRelativeToBase() throws Exception - { - PageSwapperFactory factory = createSwapperFactory(); - File base = baseDirectory(); - File sub = new File( base, "sub" ); - File a = new File( base, "a" ); - File b = new File( sub, "b" ); - mkdirs( sub ); - createSwapperAndFile( factory, a ); - createSwapperAndFile( factory, b ); - Set set = factory.streamFilesRecursive( base ).map( FileHandle::getRelativeFile ).collect( toSet() ); - assertThat( "Files relative to base directory " + base, - set, containsInAnyOrder( new File( "a" ), new File( "sub" + File.separator + "b" ) ) ); - } - - @Test - public void streamFilesRecursiveMustBeAbleToGivePathRelativeToRoot() throws Exception - { - assumeTrue( isRootAccessible() ); - assumeFalse( IS_OS_WINDOWS ); - PageSwapperFactory factory = createSwapperFactory(); - File base = new File( "/" ); - File sub = new File( base, "sub" ); - File a = new File( base, "a" ); - File b = new File( sub, "b" ); - mkdirs( sub ); - createSwapperAndFile( factory, a ); - createSwapperAndFile( factory, b ); - Set set = factory.streamFilesRecursive( base ).map( FileHandle::getRelativeFile ).collect( toSet() ); - assertThat( "Files relative to base directory " + base, - set, containsInAnyOrder( new File( "a" ), new File( "sub" + File.separator + "b" ) ) ); - } - - @Test - public void streamFilesRecursiveMustListSingleFileGivenAsBase() throws Exception - { - PageSwapperFactory factory = createSwapperFactory(); - File base = baseDirectory(); - File a = new File( base, "a" ); - File b = new File( base, "b" ); - createSwapperAndFile( factory, a ); - createSwapperAndFile( factory, b ); - Set files = factory.streamFilesRecursive( a ).map( FileHandle::getFile ).collect( toSet() ); - assertThat( files, containsInAnyOrder( a ) ); - } - - @Test - public void streamFilesRecursiveMustThrowOnNonExistingBasePath() throws Exception - { - PageSwapperFactory factory = createSwapperFactory(); - File base = baseDirectory(); - File nonExisting = new File( base, "nonExisting" ); - expectedException.expect( NoSuchFileException.class ); - factory.streamFilesRecursive( nonExisting ); - } - - @Test - public void streamFilesRecursiveMustRenameFiles() throws Exception - { - PageSwapperFactory factory = createSwapperFactory(); - File base = baseDirectory(); - File a = new File( base, "a" ); - File b = new File( base, "b" ); - createSwapperAndFile( factory, a ).close(); - FileHandle handle = factory.streamFilesRecursive( a ).findAny().get(); - handle.rename( b ); - createSwapper( factory, b, cachePageSize(), NO_CALLBACK, false ); // throws if 'b' does not exist - } - - @Test - public void streamFilesRecursiveMustRenameDelete() throws Exception - { - PageSwapperFactory factory = createSwapperFactory(); - File base = baseDirectory(); - File a = new File( base, "a" ); - File b = new File( base, "b" ); - createSwapperAndFile( factory, a ).close(); - createSwapperAndFile( factory, b ).close(); - FileHandle handle = factory.streamFilesRecursive( a ).findAny().get(); - handle.delete(); - Set files = factory.streamFilesRecursive( base ).map( FileHandle::getFile ).collect( toSet() ); - assertThat( files, containsInAnyOrder( b ) ); - } - - @Test - public void streamFilesRecursiveMustThrowWhenDeletingNonExistingFile() throws Exception - { - PageSwapperFactory factory = createSwapperFactory(); - File base = baseDirectory(); - File a = new File( base, "a" ); - PageSwapper swapperA = createSwapperAndFile( factory, a ); - FileHandle handle = factory.streamFilesRecursive( a ).findAny().get(); - swapperA.closeAndDelete(); - expectedException.expect( NoSuchFileException.class ); - handle.delete(); - } - - @Test - public void streamFilesRecursiveMustThrowWhenTargetFileOfRenameAlreadyExists() throws Exception - { - PageSwapperFactory factory = createSwapperFactory(); - File base = baseDirectory(); - File a = new File( base, "a" ); - File b = new File( base, "b" ); - createSwapperAndFile( factory, a ).close(); - createSwapperAndFile( factory, b ).close(); - FileHandle handle = factory.streamFilesRecursive( a ).findAny().get(); - expectedException.expect( FileAlreadyExistsException.class ); - handle.rename( b ); - } - - @Test - public void streamFilesRecursiveMustNotThrowWhenTargetFileOfRenameAlreadyExistsAndUsingReplaceExisting() throws Exception - { - PageSwapperFactory factory = createSwapperFactory(); - File base = baseDirectory(); - File a = new File( base, "a" ); - File b = new File( base, "b" ); - createSwapperAndFile( factory, a ).close(); - createSwapperAndFile( factory, b ).close(); - FileHandle handle = factory.streamFilesRecursive( a ).findAny().get(); - handle.rename( b, REPLACE_EXISTING ); - } - - @Test - public void streamFilesRecursiveMustCreateMissingPathDirectoriesImpliedByFileRename() throws Exception - { - PageSwapperFactory factory = createSwapperFactory(); - File base = baseDirectory(); - File a = new File( base, "a" ); - File target = new File( new File( new File( base, "sub" ), "sub" ), "target" ); - createSwapperAndFile( factory, a ).close(); - FileHandle handle = factory.streamFilesRecursive( a ).findAny().get(); - handle.rename( target ); - createSwapper( factory, target, cachePageSize(), NO_CALLBACK, false ); // must not throw - } - - @Test - public void streamFilesRecursiveMustNotSeeFilesLaterCreatedBaseDirectory() throws Exception - { - PageSwapperFactory factory = createSwapperFactory(); - File base = baseDirectory(); - File a = new File( base, "a" ); - File b = new File( base, "b" ); - createSwapperAndFile( factory, a ).close(); // note that we don't create 'b' at this point - Stream stream = factory.streamFilesRecursive( base ); // stream takes a snapshot of file tree - createSwapperAndFile( factory, b ).close(); // 'b' now exists, but it's too late to be included in snapshot - assertThat( stream.map( FileHandle::getFile ).collect( toSet() ), containsInAnyOrder( a ) ); - } - - @Test - public void streamFilesRecursiveMustNotSeeFilesRenamedIntoBaseDirectory() throws Exception - { - PageSwapperFactory factory = createSwapperFactory(); - File base = baseDirectory(); - File a = new File( base, "a" ); - File sub = new File( base, "sub" ); - mkdirs( sub ); - File x = new File( sub, "x" ); - createSwapperAndFile( factory, a ).close(); - createSwapperAndFile( factory, x ).close(); - File target = new File( base, "target" ); - Iterable handles = factory.streamFilesRecursive( base )::iterator; - Set observedFiles = new HashSet<>(); - for ( FileHandle handle : handles ) - { - File file = handle.getFile(); - observedFiles.add( file ); - if ( file.equals( x ) ) - { - handle.rename( target ); - } - } - assertThat( observedFiles, containsInAnyOrder( a, x ) ); - } - - @Test - public void streamFilesRecursiveMustNotSeeFilesRenamedIntoSubDirectory() throws Exception - { - PageSwapperFactory factory = createSwapperFactory(); - File base = baseDirectory(); - File a = new File( base, "a" ); - File sub = new File( base, "sub" ); - mkdirs( sub ); - File target = new File( sub, "target" ); - createSwapperAndFile( factory, a ).close(); - Iterable handles = factory.streamFilesRecursive( base )::iterator; - Set observedFiles = new HashSet<>(); - for ( FileHandle handle : handles ) - { - File file = handle.getFile(); - observedFiles.add( file ); - if ( file.equals( a ) ) - { - handle.rename( target ); - } - } - assertThat( observedFiles, containsInAnyOrder( a ) ); - } - - @Test - public void streamFilesRecursiveSourceFileMustNotExistAfterRename() throws Exception - { - PageSwapperFactory factory = createSwapperFactory(); - File base = baseDirectory(); - File a = new File( base, "a" ); - File b = new File( base, "b" ); - createSwapperAndFile( factory, a ).close(); - FileHandle handle = factory.streamFilesRecursive( a ).findAny().get(); - handle.rename( b ); - expectedException.expect( NoSuchFileException.class ); - createSwapper( factory, a, cachePageSize(), NO_CALLBACK, false ); // throws because 'a' no longer exists - } - - @Test - public void streamFilesRecursiveRenameMustNotChangeSourceFileContents() throws Exception - { - PageSwapperFactory factory = createSwapperFactory(); - File base = baseDirectory(); - File a = new File( base, "a" ); - File b = new File( base, "b" ); - ByteBufferPage page = createPage(); - PageSwapper swapper = createSwapperAndFile( factory, a ); - long expectedValue = 0xdeadbeeffefefeL; - page.putLong( expectedValue, 0 ); - swapper.write( 0, page ); - clear( page ); - swapper.close(); - FileHandle handle = factory.streamFilesRecursive( a ).findAny().get(); - handle.rename( b ); - swapper = createSwapper( factory, b, cachePageSize(), NO_CALLBACK, false ); - swapper.read( 0, page ); - long actualValue = page.getLong( 0 ); - assertThat( actualValue, is( expectedValue ) ); - } - - @Test - public void streamFilesRecursiveRenameMustNotChangeSourceFileContentsWithReplaceExisting() - throws Exception - { - PageSwapperFactory factory = createSwapperFactory(); - File base = baseDirectory(); - File a = new File( base, "a" ); - File b = new File( base, "b" ); - ByteBufferPage page = createPage(); - PageSwapper swapper = createSwapperAndFile( factory, a ); - long expectedValue = 0xdeadbeeffefefeL; - page.putLong( expectedValue, 0 ); - swapper.write( 0, page ); - clear( page ); - swapper.close(); - swapper = createSwapperAndFile( factory, b ); - page.putLong( ThreadLocalRandom.current().nextLong(), 0 ); - swapper.write( 0, page ); - swapper.close(); - clear( page ); - FileHandle handle = factory.streamFilesRecursive( a ).findAny().get(); - handle.rename( b, REPLACE_EXISTING ); - swapper = createSwapper( factory, b, cachePageSize(), NO_CALLBACK, false ); - swapper.read( 0, page ); - long actualValue = page.getLong( 0 ); - assertThat( actualValue, is( expectedValue ) ); - } } diff --git a/community/io/src/test/java/org/neo4j/test/rule/fs/FileSystemRule.java b/community/io/src/test/java/org/neo4j/test/rule/fs/FileSystemRule.java index 2ff0ae2cbab7d..345f067cc7beb 100644 --- a/community/io/src/test/java/org/neo4j/test/rule/fs/FileSystemRule.java +++ b/community/io/src/test/java/org/neo4j/test/rule/fs/FileSystemRule.java @@ -33,7 +33,9 @@ import java.nio.file.CopyOption; import java.util.function.Function; import java.util.function.Supplier; +import java.util.stream.Stream; +import org.neo4j.io.fs.FileHandle; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.fs.StoreChannel; import org.neo4j.io.fs.watcher.FileWatcher; @@ -219,6 +221,12 @@ public void deleteFileOrThrow( File file ) throws IOException fs.deleteFileOrThrow( file ); } + @Override + public Stream streamFilesRecursive( File directory ) throws IOException + { + return fs.streamFilesRecursive( directory ); + } + @Override public int hashCode() { diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/labelscan/NativeLabelScanStore.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/labelscan/NativeLabelScanStore.java index e30a945eb1502..bdc00937b20a8 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/labelscan/NativeLabelScanStore.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/labelscan/NativeLabelScanStore.java @@ -38,7 +38,7 @@ import org.neo4j.index.internal.gbptree.Hit; import org.neo4j.index.internal.gbptree.Layout; import org.neo4j.index.internal.gbptree.MetadataMismatchException; -import org.neo4j.io.pagecache.FileHandle; +import org.neo4j.io.fs.FileHandle; import org.neo4j.io.pagecache.IOLimiter; import org.neo4j.io.pagecache.PageCache; import org.neo4j.io.pagecache.PageCursor; @@ -349,7 +349,7 @@ public boolean hasStore() throws IOException private Optional storeFileHandle() throws IOException { - return pageCache.streamFilesRecursive( storeFile ).findFirst(); + return pageCache.getCachedFileSystem().streamFilesRecursive( storeFile ).findFirst() ; } /** diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/StoreUpgrader.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/StoreUpgrader.java index 91bf633f3c137..7901050909773 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/StoreUpgrader.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/StoreUpgrader.java @@ -30,8 +30,8 @@ import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.helpers.Exceptions; +import org.neo4j.io.fs.FileHandle; import org.neo4j.io.fs.FileSystemAbstraction; -import org.neo4j.io.pagecache.FileHandle; import org.neo4j.io.pagecache.PageCache; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.storemigration.monitoring.MigrationProgressMonitor; @@ -264,11 +264,12 @@ private void cleanMigrationDirectory( File migrationDirectory ) { fileSystem.deleteRecursively( migrationDirectory ); } - // We use the page cache here to make sure that the migration directory is clean even if we are using a - // block device. + // We use the file system from the page cache here to make sure that the migration directory is clean + // even if we are using a block device. try { - pageCache.streamFilesRecursive( migrationDirectory ).forEach( FileHandle.HANDLE_DELETE ); + pageCache.getCachedFileSystem().streamFilesRecursive( migrationDirectory ) + .forEach( FileHandle.HANDLE_DELETE ); } catch ( NoSuchFileException e ) { diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigrator.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigrator.java index b64ec36ce9f96..a53320e5496d3 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigrator.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigrator.java @@ -43,8 +43,8 @@ import java.util.stream.StreamSupport; import org.neo4j.helpers.collection.Iterables; +import org.neo4j.io.fs.FileHandle; import org.neo4j.io.fs.FileSystemAbstraction; -import org.neo4j.io.pagecache.FileHandle; import org.neo4j.io.pagecache.PageCache; import org.neo4j.io.pagecache.PageCursor; import org.neo4j.io.pagecache.PagedFile; @@ -413,13 +413,14 @@ private void migrateWithBatchImporter( File storeDir, File migrationDir, long la } StoreFile.fileOperation( DELETE, fileSystem, migrationDir, null, storesToDeleteFromMigratedDirectory, true, null, StoreFileType.values() ); - // When migrating on a block device there might be some files only accessible via the page cache. + // When migrating on a block device there might be some files only accessible via the file system + // provided by the page cache. try { Predicate fileHandlePredicate = fileHandle -> storesToDeleteFromMigratedDirectory.stream() .anyMatch( storeFile -> storeFile.fileName( StoreFileType.STORE ) .equals( fileHandle.getFile().getName() ) ); - pageCache.streamFilesRecursive( migrationDir ).filter( fileHandlePredicate ) + pageCache.getCachedFileSystem().streamFilesRecursive( migrationDir ).filter( fileHandlePredicate ) .forEach( FileHandle.HANDLE_DELETE ); } catch ( NoSuchFileException e ) @@ -640,11 +641,12 @@ public void moveMigratedFiles( File migrationDir, File storeDir, String versionT true, // allow to skip non existent source files ExistingTargetStrategy.OVERWRITE, // allow to overwrite target files StoreFileType.values() ); - // Since some of the files might only be accessible through the page cache (i.e. block devices), we also try to - // move the files with the page cache. + // Since some of the files might only be accessible through the file system provided by the page cache (i.e. + // block devices), we also try to move the files with the page cache. try { - Iterable fileHandles = pageCache.streamFilesRecursive( migrationDir )::iterator; + Iterable fileHandles = pageCache.getCachedFileSystem() + .streamFilesRecursive( migrationDir )::iterator; for ( FileHandle fh : fileHandles ) { Predicate predicate = diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/pagecache/PageSwapperFactoryForTesting.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/pagecache/PageSwapperFactoryForTesting.java index 5d59dbddc8add..1cde3e7625472 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/pagecache/PageSwapperFactoryForTesting.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/pagecache/PageSwapperFactoryForTesting.java @@ -19,14 +19,11 @@ */ package org.neo4j.kernel.impl.pagecache; -import java.io.File; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; -import java.util.stream.Stream; import org.neo4j.graphdb.config.Configuration; import org.neo4j.io.fs.FileSystemAbstraction; -import org.neo4j.io.pagecache.FileHandle; import org.neo4j.io.pagecache.PageSwapperFactory; import org.neo4j.io.pagecache.impl.SingleFilePageSwapperFactory; @@ -74,12 +71,6 @@ public boolean isCachePageSizeHintStrict() return cachePageSizeHintIsStrict.get(); } - @Override - public Stream streamFilesRecursive( File directory ) - { - return Stream.empty(); - } - @Override public void open( FileSystemAbstraction fs, Configuration configuration ) { diff --git a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/StoreFiles.java b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/StoreFiles.java index 100ae6904e54b..6e7d2dea2c347 100644 --- a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/StoreFiles.java +++ b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/StoreFiles.java @@ -29,8 +29,8 @@ import java.util.stream.Stream; import org.neo4j.causalclustering.identity.StoreId; +import org.neo4j.io.fs.FileHandle; import org.neo4j.io.fs.FileSystemAbstraction; -import org.neo4j.io.pagecache.FileHandle; import org.neo4j.io.pagecache.PageCache; import org.neo4j.kernel.impl.store.MetaDataStore; @@ -83,7 +83,7 @@ private Stream acceptedPageCachedFiles( File storeDir ) throws IOExc { try { - Stream stream = pageCache.streamFilesRecursive( storeDir ); + Stream stream = pageCache.getCachedFileSystem().streamFilesRecursive( storeDir ); Predicate acceptableFiles = fh -> fileFilter.accept( storeDir, fh.getRelativeFile().getPath() ); return stream.filter( acceptableFiles ); } diff --git a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/readreplica/ReadReplicaStartupProcessTest.java b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/readreplica/ReadReplicaStartupProcessTest.java index 27eee79d15e85..2aab515b3b965 100644 --- a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/readreplica/ReadReplicaStartupProcessTest.java +++ b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/readreplica/ReadReplicaStartupProcessTest.java @@ -23,10 +23,10 @@ import org.junit.Test; import java.io.File; +import java.io.IOException; import java.util.HashMap; import java.util.Map; import java.util.Optional; -import java.io.IOException; import java.util.UUID; import java.util.stream.Stream; @@ -41,6 +41,7 @@ import org.neo4j.causalclustering.identity.MemberId; import org.neo4j.causalclustering.identity.StoreId; import org.neo4j.helpers.Service; +import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; import org.neo4j.kernel.lifecycle.Lifecycle; import org.neo4j.logging.NullLogProvider; @@ -77,7 +78,10 @@ public void commonMocking() throws StoreIdDownloadFailedException, IOException Map members = new HashMap<>(); members.put( memberId, mock( CoreServerInfo.class ) ); - when( pageCache.streamFilesRecursive( any(File.class) ) ).thenAnswer( ( f ) -> Stream.empty() ); + FileSystemAbstraction fileSystemAbstraction = mock( FileSystemAbstraction.class ); + when( fileSystemAbstraction.streamFilesRecursive( any( File.class ) ) ) + .thenAnswer( ( f ) -> Stream.empty()); + when( pageCache.getCachedFileSystem() ).thenReturn( fileSystemAbstraction ); when( localDatabase.storeDir() ).thenReturn( storeDir ); when( localDatabase.storeId() ).thenReturn( localStoreId ); when( topologyService.coreServers() ).thenReturn( clusterTopology ); diff --git a/enterprise/com/src/main/java/org/neo4j/com/storecopy/ExternallyManagedPageCache.java b/enterprise/com/src/main/java/org/neo4j/com/storecopy/ExternallyManagedPageCache.java index dc311e4d8f3bd..7620f99eaf879 100644 --- a/enterprise/com/src/main/java/org/neo4j/com/storecopy/ExternallyManagedPageCache.java +++ b/enterprise/com/src/main/java/org/neo4j/com/storecopy/ExternallyManagedPageCache.java @@ -28,7 +28,6 @@ import org.neo4j.graphdb.GraphDatabaseService; import org.neo4j.graphdb.factory.GraphDatabaseFactory; import org.neo4j.io.fs.FileSystemAbstraction; -import org.neo4j.io.pagecache.FileHandle; import org.neo4j.io.pagecache.IOLimiter; import org.neo4j.io.pagecache.PageCache; import org.neo4j.io.pagecache.PagedFile; @@ -100,9 +99,9 @@ public int maxCachedPages() } @Override - public Stream streamFilesRecursive( File directory ) throws IOException + public FileSystemAbstraction getCachedFileSystem() { - return delegate.streamFilesRecursive( directory ); + return delegate.getCachedFileSystem(); } /** diff --git a/enterprise/com/src/main/java/org/neo4j/com/storecopy/FileMoveAction.java b/enterprise/com/src/main/java/org/neo4j/com/storecopy/FileMoveAction.java index 41db3514ed70b..a0468235cfe95 100644 --- a/enterprise/com/src/main/java/org/neo4j/com/storecopy/FileMoveAction.java +++ b/enterprise/com/src/main/java/org/neo4j/com/storecopy/FileMoveAction.java @@ -26,7 +26,7 @@ import java.nio.file.Path; import java.util.Optional; -import org.neo4j.io.pagecache.FileHandle; +import org.neo4j.io.fs.FileHandle; import org.neo4j.io.pagecache.PageCache; @FunctionalInterface @@ -38,7 +38,7 @@ static FileMoveAction copyViaPageCache( File file, PageCache pageCache ) { return ( toDir, copyOptions ) -> { - Optional handle = pageCache.streamFilesRecursive( file ).findAny(); + Optional handle = pageCache.getCachedFileSystem().streamFilesRecursive( file ).findAny(); if ( handle.isPresent() ) { handle.get().rename( new File( toDir, file.getName() ), copyOptions ); diff --git a/enterprise/com/src/main/java/org/neo4j/com/storecopy/StoreUtil.java b/enterprise/com/src/main/java/org/neo4j/com/storecopy/StoreUtil.java index 2e79237e11fa3..9dc46914a7034 100644 --- a/enterprise/com/src/main/java/org/neo4j/com/storecopy/StoreUtil.java +++ b/enterprise/com/src/main/java/org/neo4j/com/storecopy/StoreUtil.java @@ -28,10 +28,10 @@ import java.util.stream.Stream; import org.neo4j.io.fs.FileUtils; -import org.neo4j.io.pagecache.FileHandle; +import org.neo4j.io.fs.FileHandle; import org.neo4j.io.pagecache.PageCache; -import static org.neo4j.io.pagecache.FileHandle.HANDLE_DELETE; +import static org.neo4j.io.fs.FileHandle.HANDLE_DELETE; public class StoreUtil { @@ -74,7 +74,7 @@ public static void cleanStoreDir( File storeDir, PageCache pageCache ) throws IO FileUtils.deleteRecursively( file ); } - pageCache.streamFilesRecursive( storeDir ) + pageCache.getCachedFileSystem().streamFilesRecursive( storeDir ) .filter( fh -> DEEP_STORE_FILE_FILTER.accept( fh.getFile() ) ).forEach( HANDLE_DELETE ); } @@ -101,7 +101,7 @@ public static void moveAwayDbWithPageCache( File from, File to, PageCache pageCa final Stream fileHandleStream; try { - fileHandleStream = pageCache.streamFilesRecursive( from ); + fileHandleStream = pageCache.getCachedFileSystem().streamFilesRecursive( from ); } catch ( IOException e ) { @@ -115,7 +115,7 @@ public static void moveAwayDbWithPageCache( File from, File to, PageCache pageCa public static void deleteRecursive( File storeDir, PageCache pageCache ) throws IOException { FileUtils.deleteRecursively( storeDir ); - pageCache.streamFilesRecursive( storeDir ).forEach( HANDLE_DELETE ); + pageCache.getCachedFileSystem().streamFilesRecursive( storeDir ).forEach( HANDLE_DELETE ); } public static boolean isBranchedDataDirectory( File file ) diff --git a/enterprise/ha/src/test/java/org/neo4j/kernel/ha/cluster/SwitchToSlaveBranchThenCopyTest.java b/enterprise/ha/src/test/java/org/neo4j/kernel/ha/cluster/SwitchToSlaveBranchThenCopyTest.java index 4e914f3ff8d95..2b585a3341e53 100644 --- a/enterprise/ha/src/test/java/org/neo4j/kernel/ha/cluster/SwitchToSlaveBranchThenCopyTest.java +++ b/enterprise/ha/src/test/java/org/neo4j/kernel/ha/cluster/SwitchToSlaveBranchThenCopyTest.java @@ -276,7 +276,10 @@ private SwitchToSlaveBranchThenCopy newSwitchToSlaveSpy() throws Exception PagedFile pagedFileMock = mock( PagedFile.class ); when( pagedFileMock.getLastPageId() ).thenReturn( 1L ); when( pageCacheMock.map( any( File.class ), anyInt() ) ).thenReturn( pagedFileMock ); - when( pageCacheMock.streamFilesRecursive( any( File.class ) ) ).thenReturn( Stream.empty() ); + FileSystemAbstraction fileSystemAbstraction = mock( FileSystemAbstraction.class ); + when( fileSystemAbstraction.streamFilesRecursive( any( File.class ) ) ) + .thenAnswer( ( f ) -> Stream.empty() ); + when( pageCacheMock.getCachedFileSystem() ).thenReturn( fileSystemAbstraction ); StoreCopyClient storeCopyClient = mock( StoreCopyClient.class ); diff --git a/enterprise/ha/src/test/java/org/neo4j/kernel/ha/cluster/SwitchToSlaveCopyThenBranchTest.java b/enterprise/ha/src/test/java/org/neo4j/kernel/ha/cluster/SwitchToSlaveCopyThenBranchTest.java index ee60405109321..ebf8177199400 100644 --- a/enterprise/ha/src/test/java/org/neo4j/kernel/ha/cluster/SwitchToSlaveCopyThenBranchTest.java +++ b/enterprise/ha/src/test/java/org/neo4j/kernel/ha/cluster/SwitchToSlaveCopyThenBranchTest.java @@ -335,8 +335,10 @@ private SwitchToSlaveCopyThenBranch newSwitchToSlaveSpy() throws Exception StoreCopyClient storeCopyClient = mock( StoreCopyClient.class ); Stream mockStream = mock( Stream.class ); when( mockStream.filter( any( Predicate.class ) ) ).thenReturn( mock( Stream.class ) ); - when( pageCacheMock.streamFilesRecursive( any( File.class) ) ).thenReturn( mockStream ); - + FileSystemAbstraction fileSystemAbstraction = mock( FileSystemAbstraction.class ); + when( fileSystemAbstraction.streamFilesRecursive( any( File.class ) ) ) + .thenReturn( mockStream ); + when( pageCacheMock.getCachedFileSystem() ).thenReturn( fileSystemAbstraction ); return newSwitchToSlaveSpy( pageCacheMock, storeCopyClient ); }