From 5ca441acaa44b88888a90ce2259b4f381fd271f8 Mon Sep 17 00:00:00 2001 From: MishaDemianenko Date: Mon, 7 May 2018 18:32:56 +0200 Subject: [PATCH] Remove custom block device support. Remove page cache file system. Remove ability to stream files over page cache. Always use default file system for all copy/transfer operations. Remove page file read and write channel. --- .../consistency/ConsistencyCheckService.java | 2 +- .../checking/GraphStoreFixture.java | 4 +- .../org/neo4j/io/pagecache/PageCache.java | 19 --- .../io/pagecache/PageSwapperFactory.java | 5 - .../org/neo4j/io/pagecache/PagedFile.java | 32 ----- .../impl/PagedReadableByteChannel.java | 93 ------------ .../impl/PagedWritableByteChannel.java | 91 ------------ .../impl/SingleFilePageSwapperFactory.java | 6 - .../impl/muninn/MuninnPageCache.java | 15 -- .../impl/muninn/MuninnPagedFile.java | 16 --- .../pagecache/AdversarialPageCache.java | 13 -- .../pagecache/AdversarialPagedFile.java | 16 --- .../io/pagecache/DelegatingPageCache.java | 13 -- .../io/pagecache/DelegatingPagedFile.java | 14 -- .../org/neo4j/io/pagecache/PageCacheTest.java | 135 ++---------------- .../org/neo4j/io/pagecache/StubPagedFile.java | 14 -- .../pagecache/impl/PagedByteChannelsTest.java | 77 ---------- .../org/neo4j/kernel/NeoStoreDataSource.java | 6 +- .../labelscan/GBPTreePageCacheFileUtil.java | 90 ------------ .../index/labelscan/NativeLabelScanStore.java | 10 +- .../impl/index/schema/NativeSchemaIndex.java | 2 - .../recordstorage/RecordStorageEngine.java | 2 +- .../neo4j/kernel/impl/store/StoreFactory.java | 2 +- .../neo4j/kernel/impl/store/StoreType.java | 2 - .../store/format/RecordFormatSelector.java | 23 +-- .../impl/storemigration/StoreUpgrader.java | 15 +- .../NativeLabelScanStoreMigrator.java | 11 +- .../participant/StoreMigrator.java | 97 +------------ .../kernel/internal/KernelDiagnostics.java | 2 +- .../internal/BatchInserterImpl.java | 4 +- .../batchimport/store/BatchingNeoStores.java | 5 +- .../index/AbstractGBPTreeFileUtilTest.java | 3 - .../impl/index/GBPTreeFileUtilTest.java | 5 +- .../labelscan/NativeLabelScanStoreIT.java | 5 +- .../NativeLabelScanStoreRebuildTest.java | 10 +- .../labelscan/NativeLabelScanStoreTest.java | 2 +- .../kernel/impl/store/StoreFactoryTest.java | 2 +- .../NativeLabelScanStoreMigratorTest.java | 2 +- .../participant/StoreMigratorTest.java | 6 +- .../batchinsert/internal/BatchInsertTest.java | 6 +- .../backup/OnlineBackupKernelExtension.java | 2 +- .../neo4j/backup/impl/BackupCopyService.java | 2 +- .../BackupStrategyCoordinatorFactory.java | 2 +- .../catchup/RegularCatchupServerHandler.java | 13 +- .../storecopy/PrepareStoreCopyFiles.java | 7 +- .../PrepareStoreCopyFilesProvider.java | 9 +- .../catchup/storecopy/RemoteStore.java | 2 +- .../storecopy/StoreCopyRequestHandler.java | 16 +-- .../catchup/storecopy/StoreFiles.java | 2 +- .../catchup/storecopy/StoreResource.java | 19 +-- .../storecopy/StoreResourceStreamFactory.java | 7 +- .../catchup/storecopy/StreamToDisk.java | 6 - .../storecopy/StreamToDiskProvider.java | 19 +-- .../core/server/CoreServerModule.java | 4 +- .../EnterpriseReadReplicaEditionModule.java | 2 +- .../catchup/storecopy/FakeCatchupServer.java | 5 +- .../catchup/storecopy/FileSenderTest.java | 18 +-- .../storecopy/PrepareStoreCopyFilesTest.java | 8 +- .../storecopy/SimpleCatchupClient.java | 2 +- .../catchup/storecopy/StoreCopyClientIT.java | 48 ++----- .../StoreCopyRequestHandlerTest.java | 21 ++- .../StoreFileStreamingProtocolTest.java | 12 +- .../catchup/storecopy/StoreFilesTest.java | 73 ++-------- .../StoreFilesWithRealFileSystemTest.java | 3 - .../catchup/storecopy/StreamToDiskTest.java | 23 ++- .../catchup/storecopy/TestCatchupServer.java | 5 +- .../ReadReplicaStartupProcessTest.java | 1 - .../scenarios/CoreToCoreCopySnapshotIT.java | 4 - .../scenarios/ReadReplicaReplicationIT.java | 6 +- .../storecopy/ExternallyManagedPageCache.java | 12 -- .../neo4j/com/storecopy/FileMoveAction.java | 27 ---- .../neo4j/com/storecopy/FileMoveProvider.java | 37 +---- .../neo4j/com/storecopy/StoreCopyClient.java | 22 +-- .../neo4j/com/storecopy/StoreCopyServer.java | 28 +--- .../org/neo4j/com/storecopy/StoreUtil.java | 49 +------ .../com/storecopy/ToFileStoreWriter.java | 51 +------ .../com/storecopy/FileMoveProviderTest.java | 57 +------- .../com/storecopy/StoreCopyClientTest.java | 29 +--- .../com/storecopy/ToFileStoreWriterTest.java | 29 ++-- .../neo4j/kernel/ha/BranchedDataMigrator.java | 3 +- .../neo4j/kernel/ha/BranchedDataPolicy.java | 8 +- .../ha/cluster/DefaultMasterImplSPI.java | 5 +- .../kernel/ha/cluster/SwitchToSlave.java | 2 +- .../SwitchToSlaveBranchThenCopyTest.java | 1 - .../SwitchToSlaveCopyThenBranchTest.java | 1 - .../format/RecordFormatSelectorTest.java | 41 +++--- .../impl/storemigration/StoreMigrationIT.java | 2 - .../upgrade/RecordFormatsMigrationIT.java | 2 +- 88 files changed, 213 insertions(+), 1411 deletions(-) delete mode 100644 community/io/src/main/java/org/neo4j/io/pagecache/impl/PagedReadableByteChannel.java delete mode 100644 community/io/src/main/java/org/neo4j/io/pagecache/impl/PagedWritableByteChannel.java delete mode 100644 community/io/src/test/java/org/neo4j/io/pagecache/impl/PagedByteChannelsTest.java delete mode 100644 community/kernel/src/main/java/org/neo4j/kernel/impl/index/labelscan/GBPTreePageCacheFileUtil.java diff --git a/community/consistency-check/src/main/java/org/neo4j/consistency/ConsistencyCheckService.java b/community/consistency-check/src/main/java/org/neo4j/consistency/ConsistencyCheckService.java index bda2442081567..2fc6b7de4d19b 100644 --- a/community/consistency-check/src/main/java/org/neo4j/consistency/ConsistencyCheckService.java +++ b/community/consistency-check/src/main/java/org/neo4j/consistency/ConsistencyCheckService.java @@ -244,7 +244,7 @@ fileSystem, config, new SimpleLogService( logProvider, logProvider ), pageCache, IndexProviderMap indexes = loadIndexProviders( extensions ); LabelScanStore labelScanStore = - new NativeLabelScanStore( pageCache, storeDir, FullStoreChangeStream.EMPTY, true, monitors, + new NativeLabelScanStore( pageCache, storeDir, fileSystem, FullStoreChangeStream.EMPTY, true, monitors, RecoveryCleanupWorkCollector.IGNORE ); life.add( labelScanStore ); diff --git a/community/consistency-check/src/test/java/org/neo4j/consistency/checking/GraphStoreFixture.java b/community/consistency-check/src/test/java/org/neo4j/consistency/checking/GraphStoreFixture.java index 9e87886b4b1e8..b0bd8da33f35c 100644 --- a/community/consistency-check/src/test/java/org/neo4j/consistency/checking/GraphStoreFixture.java +++ b/community/consistency-check/src/test/java/org/neo4j/consistency/checking/GraphStoreFixture.java @@ -39,13 +39,13 @@ import org.neo4j.graphdb.factory.GraphDatabaseBuilder; import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector; +import org.neo4j.internal.kernel.api.exceptions.TransactionFailureException; import org.neo4j.io.fs.DefaultFileSystemAbstraction; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.api.StatementConstants; import org.neo4j.kernel.api.direct.DirectStoreAccess; -import org.neo4j.internal.kernel.api.exceptions.TransactionFailureException; import org.neo4j.kernel.api.labelscan.LabelScanStore; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.extension.KernelExtensions; @@ -197,7 +197,7 @@ public DirectStoreAccess directStoreAccess() private LabelScanStore startLabelScanStore( PageCache pageCache, IndexStoreView indexStoreView, Monitors monitors ) { NativeLabelScanStore labelScanStore = - new NativeLabelScanStore( pageCache, directory, new FullLabelStream( indexStoreView ), false, monitors, + new NativeLabelScanStore( pageCache, directory, fileSystem, new FullLabelStream( indexStoreView ), false, monitors, RecoveryCleanupWorkCollector.IMMEDIATE ); try { diff --git a/community/io/src/main/java/org/neo4j/io/pagecache/PageCache.java b/community/io/src/main/java/org/neo4j/io/pagecache/PageCache.java index 83bcd08740079..a23de20e6dae1 100644 --- a/community/io/src/main/java/org/neo4j/io/pagecache/PageCache.java +++ b/community/io/src/main/java/org/neo4j/io/pagecache/PageCache.java @@ -26,8 +26,6 @@ import java.util.List; import java.util.Optional; -import org.neo4j.io.fs.FileSystemAbstraction; - /** * A page caching mechanism that allows caching multiple files and accessing their data * in pages via a re-usable cursor. @@ -134,26 +132,9 @@ public interface PageCache extends AutoCloseable */ long maxCachedPages(); - /** - * Get the {@link FileSystemAbstraction} that represents the filesystem where the paged files reside. - * - * @return the filesystem that the page cache is using. - */ - FileSystemAbstraction getCachedFileSystem(); - /** * Report any thread-local events to the global page cache tracer, as if acquiring a thread-specific page cursor * tracer, and reporting the events collected within it. */ void reportEvents(); - - /** - * Check if the backing {@link FileSystemAbstraction file system} supports regular file operations or not. - *

- * E.g. the file system for block device will not work with generic open and read/write calls and all operations - * needs to be done through the page cache. - * - * @return {@code true} if the backing file system supports regular file operations. - */ - boolean fileSystemSupportsFileOperations(); } diff --git a/community/io/src/main/java/org/neo4j/io/pagecache/PageSwapperFactory.java b/community/io/src/main/java/org/neo4j/io/pagecache/PageSwapperFactory.java index e7991a77fff23..02578e27cd0e0 100644 --- a/community/io/src/main/java/org/neo4j/io/pagecache/PageSwapperFactory.java +++ b/community/io/src/main/java/org/neo4j/io/pagecache/PageSwapperFactory.java @@ -45,11 +45,6 @@ public interface PageSwapperFactory */ void open( FileSystemAbstraction fs, Configuration config ); - /** - * Get the {@link FileSystemAbstraction} that represents the underlying storage for the page swapper. - */ - FileSystemAbstraction getFileSystemAbstraction(); - /** * Get the name of this PageSwapperFactory implementation, for configuration purpose. */ diff --git a/community/io/src/main/java/org/neo4j/io/pagecache/PagedFile.java b/community/io/src/main/java/org/neo4j/io/pagecache/PagedFile.java index aa76fefac99de..699a7b64c96ff 100644 --- a/community/io/src/main/java/org/neo4j/io/pagecache/PagedFile.java +++ b/community/io/src/main/java/org/neo4j/io/pagecache/PagedFile.java @@ -21,9 +21,6 @@ import java.io.File; import java.io.IOException; -import java.nio.channels.ReadableByteChannel; -import java.nio.channels.WritableByteChannel; -import java.nio.file.OpenOption; /** * The representation of a file that has been mapped into the associated page cache. @@ -191,33 +188,4 @@ public interface PagedFile extends AutoCloseable @Override void close() throws IOException; - /** - * Open a {@link ReadableByteChannel} view of this PagedFile. - *

- * The channel will read the file sequentially from the beginning till the end. - * Seeking is not supported. - *

- * The channel is not thread-safe. - * - * @return A channel for reading the paged file. - */ - ReadableByteChannel openReadableByteChannel() throws IOException; - - /** - * Open a {@link WritableByteChannel} view of this PagedFile. - *

- * The channel will write to the file sequentially from the beginning of the file, overwriting whatever is there - * already. If the amount of new data is less than the amount of existing data, then the old data will still be - * present at the far end of the file. Thus, this function works neither like opening a file for writing, nor like - * appending to a file. - *

- * If this is undesired, then the file can be mapped with {@link java.nio.file.StandardOpenOption#TRUNCATE_EXISTING} - * to remove the existing data before writing to the file. - *

- * The channel is not thread-safe. - * - * @return A channel for writing to the paged file. - * @see PageCache#map(File, int, OpenOption...) - */ - WritableByteChannel openWritableByteChannel() throws IOException; } diff --git a/community/io/src/main/java/org/neo4j/io/pagecache/impl/PagedReadableByteChannel.java b/community/io/src/main/java/org/neo4j/io/pagecache/impl/PagedReadableByteChannel.java deleted file mode 100644 index 71745366052e6..0000000000000 --- a/community/io/src/main/java/org/neo4j/io/pagecache/impl/PagedReadableByteChannel.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Copyright (c) 2002-2018 "Neo Technology," - * Network Engine for Objects in Lund AB [http://neotechnology.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package org.neo4j.io.pagecache.impl; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.channels.ClosedChannelException; -import java.nio.channels.ReadableByteChannel; - -import org.neo4j.io.pagecache.PageCursor; -import org.neo4j.io.pagecache.PagedFile; - -/** - * Presents a {@link ReadableByteChannel} view of a {@link PagedFile}. - *

- * The byte channel will read the whole file sequentially from the beginning till the end. - */ -public final class PagedReadableByteChannel implements ReadableByteChannel -{ - private final PageCursor cursor; - private boolean open = true; - private int bytesLeftInCurrentPage; - - public PagedReadableByteChannel( PagedFile pagedFile ) throws IOException - { - cursor = pagedFile.io( 0, PagedFile.PF_SHARED_READ_LOCK | PagedFile.PF_READ_AHEAD ); - } - - @Override - public int read( ByteBuffer dst ) throws IOException - { - if ( !open ) - { - throw new ClosedChannelException(); - } - if ( bytesLeftInCurrentPage == 0 ) - { - if ( cursor.next() ) - { - bytesLeftInCurrentPage = cursor.getCurrentPageSize(); - } - else - { - return -1; - } - } - int position = dst.position(); - int remaining = Math.min( dst.remaining(), bytesLeftInCurrentPage ); - int offset = cursor.getOffset(); - do - { - dst.position( position ); - cursor.setOffset( offset ); - for ( int i = 0; i < remaining; i++ ) - { - dst.put( cursor.getByte() ); - } - } - while ( cursor.shouldRetry() ); - bytesLeftInCurrentPage -= remaining; - return remaining; - } - - @Override - public boolean isOpen() - { - return open; - } - - @Override - public void close() - { - open = false; - cursor.close(); - } -} diff --git a/community/io/src/main/java/org/neo4j/io/pagecache/impl/PagedWritableByteChannel.java b/community/io/src/main/java/org/neo4j/io/pagecache/impl/PagedWritableByteChannel.java deleted file mode 100644 index efedf1a303f38..0000000000000 --- a/community/io/src/main/java/org/neo4j/io/pagecache/impl/PagedWritableByteChannel.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Copyright (c) 2002-2018 "Neo Technology," - * Network Engine for Objects in Lund AB [http://neotechnology.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package org.neo4j.io.pagecache.impl; - -import java.io.File; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.channels.ClosedChannelException; -import java.nio.channels.WritableByteChannel; -import java.nio.file.OpenOption; - -import org.neo4j.io.pagecache.PageCursor; -import org.neo4j.io.pagecache.PagedFile; - -/** - * Presents a {@link WritableByteChannel} view of the {@link PagedFile}. - *

- * The paged file will be overwritten sequentially, from the beginning till the end. - *

- * If the file already contains data, and the channel is not given enough data to overwrite the file completely, - * then the data at the end of the file will be left untouched. - *

- * If this is undesired, then the file can be mapped with {@link java.nio.file.StandardOpenOption#TRUNCATE_EXISTING} - * to remove the existing data before writing to the file. - * @see org.neo4j.io.pagecache.PageCache#map(File, int, OpenOption...) - */ -public final class PagedWritableByteChannel implements WritableByteChannel -{ - private final PageCursor cursor; - private boolean open = true; - private int bytesLeftInCurrentPage; - - public PagedWritableByteChannel( PagedFile pagedFile ) throws IOException - { - cursor = pagedFile.io( 0, PagedFile.PF_SHARED_WRITE_LOCK ); - } - - @Override - public int write( ByteBuffer src ) throws IOException - { - if ( !open ) - { - throw new ClosedChannelException(); - } - if ( bytesLeftInCurrentPage == 0 ) - { - if ( !cursor.next() ) - { - throw new IOException( "Could not advance write cursor" ); - } - bytesLeftInCurrentPage = cursor.getCurrentPageSize(); - } - int remaining = Math.min( src.remaining(), bytesLeftInCurrentPage ); - for ( int i = 0; i < remaining; i++ ) - { - cursor.putByte( src.get() ); - } - bytesLeftInCurrentPage -= remaining; - return remaining; - } - - @Override - public boolean isOpen() - { - return open; - } - - @Override - public void close() - { - open = false; - cursor.close(); - } -} diff --git a/community/io/src/main/java/org/neo4j/io/pagecache/impl/SingleFilePageSwapperFactory.java b/community/io/src/main/java/org/neo4j/io/pagecache/impl/SingleFilePageSwapperFactory.java index 04205c5c95245..6413ee31bee98 100644 --- a/community/io/src/main/java/org/neo4j/io/pagecache/impl/SingleFilePageSwapperFactory.java +++ b/community/io/src/main/java/org/neo4j/io/pagecache/impl/SingleFilePageSwapperFactory.java @@ -44,12 +44,6 @@ public void open( FileSystemAbstraction fs, Configuration config ) this.fs = fs; } - @Override - public FileSystemAbstraction getFileSystemAbstraction() - { - return fs; - } - @Override public PageSwapper createPageSwapper( File file, diff --git a/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/MuninnPageCache.java b/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/MuninnPageCache.java index a86a14d64e387..58f2f6ab6d377 100644 --- a/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/MuninnPageCache.java +++ b/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/MuninnPageCache.java @@ -35,8 +35,6 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.locks.LockSupport; -import org.neo4j.io.fs.DefaultFileSystemAbstraction; -import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.mem.MemoryAllocator; import org.neo4j.io.pagecache.IOLimiter; import org.neo4j.io.pagecache.PageCache; @@ -700,25 +698,12 @@ public long maxCachedPages() return pages.getPageCount(); } - @Override - public FileSystemAbstraction getCachedFileSystem() - { - return swapperFactory.getFileSystemAbstraction(); - } - @Override public void reportEvents() { pageCursorTracerSupplier.get().reportEvents(); } - @Override - public boolean fileSystemSupportsFileOperations() - { - // Default filesystem supports direct file access. - return getCachedFileSystem() instanceof DefaultFileSystemAbstraction; - } - int getPageCacheId() { return pageCacheId; diff --git a/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/MuninnPagedFile.java b/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/MuninnPagedFile.java index d04bc03936934..d3f2fe5130615 100644 --- a/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/MuninnPagedFile.java +++ b/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/MuninnPagedFile.java @@ -23,8 +23,6 @@ import java.io.Flushable; import java.io.IOException; import java.nio.channels.ClosedChannelException; -import java.nio.channels.ReadableByteChannel; -import java.nio.channels.WritableByteChannel; import java.util.Arrays; import org.neo4j.io.pagecache.IOLimiter; @@ -34,8 +32,6 @@ import org.neo4j.io.pagecache.PageSwapperFactory; import org.neo4j.io.pagecache.PagedFile; import org.neo4j.io.pagecache.impl.FileIsNotMappedException; -import org.neo4j.io.pagecache.impl.PagedReadableByteChannel; -import org.neo4j.io.pagecache.impl.PagedWritableByteChannel; import org.neo4j.io.pagecache.tracing.FlushEvent; import org.neo4j.io.pagecache.tracing.FlushEventOpportunity; import org.neo4j.io.pagecache.tracing.MajorFlushEvent; @@ -228,18 +224,6 @@ public void close() pageCache.unmap( this ); } - @Override - public ReadableByteChannel openReadableByteChannel() throws IOException - { - return new PagedReadableByteChannel( this ); - } - - @Override - public WritableByteChannel openWritableByteChannel() throws IOException - { - return new PagedWritableByteChannel( this ); - } - void closeSwapper() throws IOException { // We don't set closeStackTrace in close(), because the reference count may keep the file open. diff --git a/community/io/src/test/java/org/neo4j/adversaries/pagecache/AdversarialPageCache.java b/community/io/src/test/java/org/neo4j/adversaries/pagecache/AdversarialPageCache.java index 7597735473d92..d0a75bdcd097e 100644 --- a/community/io/src/test/java/org/neo4j/adversaries/pagecache/AdversarialPageCache.java +++ b/community/io/src/test/java/org/neo4j/adversaries/pagecache/AdversarialPageCache.java @@ -31,7 +31,6 @@ import java.util.Optional; import org.neo4j.adversaries.Adversary; -import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.IOLimiter; import org.neo4j.io.pagecache.PageCache; import org.neo4j.io.pagecache.PagedFile; @@ -123,21 +122,9 @@ public long maxCachedPages() return delegate.maxCachedPages(); } - @Override - public FileSystemAbstraction getCachedFileSystem() - { - return delegate.getCachedFileSystem(); - } - @Override public void reportEvents() { delegate.reportEvents(); } - - @Override - public boolean fileSystemSupportsFileOperations() - { - return delegate.fileSystemSupportsFileOperations(); - } } diff --git a/community/io/src/test/java/org/neo4j/adversaries/pagecache/AdversarialPagedFile.java b/community/io/src/test/java/org/neo4j/adversaries/pagecache/AdversarialPagedFile.java index 397859f1a98db..d5c4edc4b21c1 100644 --- a/community/io/src/test/java/org/neo4j/adversaries/pagecache/AdversarialPagedFile.java +++ b/community/io/src/test/java/org/neo4j/adversaries/pagecache/AdversarialPagedFile.java @@ -22,16 +22,12 @@ import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; -import java.nio.channels.ReadableByteChannel; -import java.nio.channels.WritableByteChannel; import java.util.Objects; import org.neo4j.adversaries.Adversary; import org.neo4j.io.pagecache.IOLimiter; import org.neo4j.io.pagecache.PageCursor; import org.neo4j.io.pagecache.PagedFile; -import org.neo4j.io.pagecache.impl.PagedReadableByteChannel; -import org.neo4j.io.pagecache.impl.PagedWritableByteChannel; /** * A {@linkplain PagedFile paged file} that wraps another paged file and an {@linkplain Adversary adversary} to provide @@ -110,16 +106,4 @@ public void close() throws IOException adversary.injectFailure( FileNotFoundException.class, IOException.class, SecurityException.class ); delegate.close(); } - - @Override - public ReadableByteChannel openReadableByteChannel() throws IOException - { - return new PagedReadableByteChannel( this ); - } - - @Override - public WritableByteChannel openWritableByteChannel() throws IOException - { - return new PagedWritableByteChannel( this ); - } } diff --git a/community/io/src/test/java/org/neo4j/io/pagecache/DelegatingPageCache.java b/community/io/src/test/java/org/neo4j/io/pagecache/DelegatingPageCache.java index 7ba3b310ec708..76ba2bc316e88 100644 --- a/community/io/src/test/java/org/neo4j/io/pagecache/DelegatingPageCache.java +++ b/community/io/src/test/java/org/neo4j/io/pagecache/DelegatingPageCache.java @@ -25,8 +25,6 @@ import java.util.List; import java.util.Optional; -import org.neo4j.io.fs.FileSystemAbstraction; - public class DelegatingPageCache implements PageCache { private final PageCache delegate; @@ -72,12 +70,6 @@ public long maxCachedPages() return delegate.maxCachedPages(); } - @Override - public FileSystemAbstraction getCachedFileSystem() - { - return delegate.getCachedFileSystem(); - } - @Override public void reportEvents() { @@ -96,9 +88,4 @@ public void flushAndForce() throws IOException delegate.flushAndForce(); } - @Override - public boolean fileSystemSupportsFileOperations() - { - return delegate.fileSystemSupportsFileOperations(); - } } diff --git a/community/io/src/test/java/org/neo4j/io/pagecache/DelegatingPagedFile.java b/community/io/src/test/java/org/neo4j/io/pagecache/DelegatingPagedFile.java index 5ce252109e35c..b8268fc3d139d 100644 --- a/community/io/src/test/java/org/neo4j/io/pagecache/DelegatingPagedFile.java +++ b/community/io/src/test/java/org/neo4j/io/pagecache/DelegatingPagedFile.java @@ -21,8 +21,6 @@ import java.io.File; import java.io.IOException; -import java.nio.channels.ReadableByteChannel; -import java.nio.channels.WritableByteChannel; public class DelegatingPagedFile implements PagedFile { @@ -75,18 +73,6 @@ public void close() throws IOException delegate.close(); } - @Override - public ReadableByteChannel openReadableByteChannel() throws IOException - { - return delegate.openReadableByteChannel(); - } - - @Override - public WritableByteChannel openWritableByteChannel() throws IOException - { - return delegate.openWritableByteChannel(); - } - @Override public void flushAndForce( IOLimiter limiter ) throws IOException { diff --git a/community/io/src/test/java/org/neo4j/io/pagecache/PageCacheTest.java b/community/io/src/test/java/org/neo4j/io/pagecache/PageCacheTest.java index f975ce1197e67..6c50a6f9a5c45 100644 --- a/community/io/src/test/java/org/neo4j/io/pagecache/PageCacheTest.java +++ b/community/io/src/test/java/org/neo4j/io/pagecache/PageCacheTest.java @@ -33,9 +33,6 @@ import java.io.UncheckedIOException; import java.nio.ByteBuffer; import java.nio.ReadOnlyBufferException; -import java.nio.channels.ClosedChannelException; -import java.nio.channels.ReadableByteChannel; -import java.nio.channels.WritableByteChannel; import java.nio.file.NoSuchFileException; import java.nio.file.OpenOption; import java.nio.file.StandardOpenOption; @@ -52,8 +49,6 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; -import org.neo4j.adversaries.RandomAdversary; -import org.neo4j.adversaries.pagecache.AdversarialPagedFile; import org.neo4j.concurrent.BinaryLatch; import org.neo4j.function.ThrowingConsumer; import org.neo4j.graphdb.config.Configuration; @@ -4225,9 +4220,10 @@ public void fileMappedWithDeleteOnCloseShouldNotFlushDirtyPagesOnClose() throws try ( PageCache cache = createPageCache( swapperFactory, maxPages, PageCacheTracer.NULL, PageCursorTracerSupplier.NULL, EmptyVersionContextSupplier.EMPTY ); PagedFile pf = cache.map( file, filePageSize, DELETE_ON_CLOSE ); - WritableByteChannel channel = pf.openWritableByteChannel() ) + PageCursor cursor = pf.io( 0, PF_SHARED_WRITE_LOCK ) ) { - generateFileWithRecords( channel, recordCount, recordSize ); + writeRecords( cursor ); + assertTrue( cursor.next() ); } assertThat( flushCounter.get(), lessThan( recordCount / recordsPerFilePage ) ); } @@ -4242,11 +4238,12 @@ public void mustFlushAllDirtyPagesWhenClosingPagedFileThatIsNotMappedWithDeleteO try ( PageCache cache = createPageCache( swapperFactory, maxPages, PageCacheTracer.NULL, PageCursorTracerSupplier.NULL, EmptyVersionContextSupplier.EMPTY ); PagedFile pf = cache.map( file, filePageSize ); - WritableByteChannel channel = pf.openWritableByteChannel() ) + PageCursor cursor = pf.io( 0, PF_SHARED_WRITE_LOCK ) ) { - generateFileWithRecords( channel, recordCount, recordSize ); + writeRecords( cursor ); + assertTrue( cursor.next() ); } - assertThat( flushCounter.get(), is( recordCount / recordsPerFilePage ) ); + assertThat( flushCounter.get(), is( 1 ) ); } private SingleFilePageSwapperFactory flushCountingPageSwapperFactory( AtomicInteger flushCounter ) @@ -4284,15 +4281,15 @@ public void fileMappedWithDeleteOnCloseMustNotLeakDirtyPages() throws Exception { configureStandardPageCache(); File file = file( "a" ); - int records = maxPages * recordsPerFilePage; int iterations = 50; for ( int i = 0; i < iterations; i++ ) { ensureExists( file ); try ( PagedFile pf = pageCache.map( file, filePageSize, DELETE_ON_CLOSE ); - WritableByteChannel channel = pf.openWritableByteChannel() ) + PageCursor cursor = pf.io( 0, PF_SHARED_WRITE_LOCK ) ) { - generateFileWithRecords( channel, records, recordSize ); + writeRecords( cursor ); + assertTrue( cursor.next() ); } } } @@ -5649,118 +5646,6 @@ public void clearCursorExceptionMustUnsetErrorConditionOnLinkedCursor() throws E } } - @Test( timeout = SHORT_TIMEOUT_MILLIS ) - public void readableByteChannelMustBeOpenUntilClosed() throws Exception - { - configureStandardPageCache(); - try ( PagedFile pf = pageCache.map( file( "a" ), filePageSize ) ) - { - ReadableByteChannel channel; - try ( ReadableByteChannel ch = pf.openReadableByteChannel() ) - { - assertTrue( ch.isOpen() ); - channel = ch; - } - assertFalse( channel.isOpen() ); - } - } - - @Test( timeout = SHORT_TIMEOUT_MILLIS ) - public void readableByteChannelMustReadAllBytesInFile() throws Exception - { - configureStandardPageCache(); - File file = file( "a" ); - generateFileWithRecords( file, recordCount, recordSize ); - try ( PagedFile pf = pageCache.map( file, filePageSize ); - ReadableByteChannel channel = pf.openReadableByteChannel() ) - { - verifyRecordsInFile( channel, recordCount ); - } - } - - @RepeatRule.Repeat( times = 5 ) - @Test( timeout = SHORT_TIMEOUT_MILLIS ) - public void readableByteChannelMustReadAllBytesInFileConsistently() throws Exception - { - configureStandardPageCache(); - File file = file( "a" ); - generateFileWithRecords( file, recordCount, recordSize ); - try ( PagedFile pf = pageCache.map( file, filePageSize ) ) - { - RandomAdversary adversary = new RandomAdversary( 0.9, 0, 0 ); - AdversarialPagedFile apf = new AdversarialPagedFile( pf, adversary ); - try ( ReadableByteChannel channel = apf.openReadableByteChannel() ) - { - verifyRecordsInFile( channel, recordCount ); - } - } - } - - @Test( timeout = SHORT_TIMEOUT_MILLIS ) - public void readingFromClosedReadableByteChannelMustThrow() throws Exception - { - File file = file( "a" ); - generateFileWithRecords( file, recordCount, recordSize ); - configureStandardPageCache(); - try ( PagedFile pf = pageCache.map( file, filePageSize ) ) - { - ReadableByteChannel channel = pf.openReadableByteChannel(); - channel.close(); - expectedException.expect( ClosedChannelException.class ); - channel.read( ByteBuffer.allocate( recordSize ) ); - fail( "That read should have thrown" ); - } - } - - @Test( timeout = SHORT_TIMEOUT_MILLIS ) - public void writableByteChannelMustBeOpenUntilClosed() throws Exception - { - configureStandardPageCache(); - try ( PagedFile pf = pageCache.map( file( "a" ), filePageSize ) ) - { - WritableByteChannel channel; - try ( WritableByteChannel ch = pf.openWritableByteChannel() ) - { - assertTrue( ch.isOpen() ); - channel = ch; - } - assertFalse( channel.isOpen() ); - } - } - - @Test( timeout = SHORT_TIMEOUT_MILLIS ) - public void writableByteChannelMustWriteAllBytesInFile() throws Exception - { - File file = file( "a" ); - configureStandardPageCache(); - try ( PagedFile pf = pageCache.map( file, filePageSize ) ) - { - try ( WritableByteChannel channel = pf.openWritableByteChannel() ) - { - generateFileWithRecords( channel, recordCount, recordSize ); - } - try ( ReadableByteChannel channel = pf.openReadableByteChannel() ) - { - verifyRecordsInFile( channel, recordCount ); - } - } - } - - @Test( timeout = SHORT_TIMEOUT_MILLIS ) - public void writingToClosedWritableByteChannelMustThrow() throws Exception - { - File file = file( "a" ); - configureStandardPageCache(); - try ( PagedFile pf = pageCache.map( file, filePageSize ) ) - { - WritableByteChannel channel = pf.openWritableByteChannel(); - channel.close(); - expectedException.expect( ClosedChannelException.class ); - channel.write( ByteBuffer.allocate( recordSize ) ); - fail( "That read should have thrown" ); - } - } - @Test public void sizeOfEmptyFileMustBeZero() throws Exception { diff --git a/community/io/src/test/java/org/neo4j/io/pagecache/StubPagedFile.java b/community/io/src/test/java/org/neo4j/io/pagecache/StubPagedFile.java index be33ca00c3eaa..e34966616564d 100644 --- a/community/io/src/test/java/org/neo4j/io/pagecache/StubPagedFile.java +++ b/community/io/src/test/java/org/neo4j/io/pagecache/StubPagedFile.java @@ -21,8 +21,6 @@ import java.io.File; import java.io.IOException; -import java.nio.channels.ReadableByteChannel; -import java.nio.channels.WritableByteChannel; public class StubPagedFile implements PagedFile { @@ -91,16 +89,4 @@ public long getLastPageId() public void close() { } - - @Override - public ReadableByteChannel openReadableByteChannel() - { - throw new UnsupportedOperationException( "Not implemented for StubPagedFile" ); - } - - @Override - public WritableByteChannel openWritableByteChannel() - { - throw new UnsupportedOperationException( "Not implemented for StubPagedFile" ); - } } diff --git a/community/io/src/test/java/org/neo4j/io/pagecache/impl/PagedByteChannelsTest.java b/community/io/src/test/java/org/neo4j/io/pagecache/impl/PagedByteChannelsTest.java deleted file mode 100644 index 14bb7ac4e2b8c..0000000000000 --- a/community/io/src/test/java/org/neo4j/io/pagecache/impl/PagedByteChannelsTest.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright (c) 2002-2018 "Neo Technology," - * Network Engine for Objects in Lund AB [http://neotechnology.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package org.neo4j.io.pagecache.impl; - -import org.junit.experimental.theories.DataPoint; -import org.junit.experimental.theories.Theories; -import org.junit.experimental.theories.Theory; -import org.junit.runner.RunWith; - -import java.io.IOException; -import java.nio.channels.Channel; -import java.nio.channels.ReadableByteChannel; -import java.nio.channels.WritableByteChannel; -import java.util.concurrent.atomic.AtomicInteger; - -import org.neo4j.function.ThrowingFunction; -import org.neo4j.io.pagecache.PageCache; -import org.neo4j.io.pagecache.PageCursor; -import org.neo4j.io.pagecache.PagedFile; -import org.neo4j.io.pagecache.StubPagedFile; - -import static org.hamcrest.Matchers.is; -import static org.junit.Assert.assertThat; - -@RunWith( Theories.class ) -public class PagedByteChannelsTest -{ - @DataPoint - public static final ThrowingFunction readable = - PagedReadableByteChannel::new; - @DataPoint - public static final ThrowingFunction writable = - PagedWritableByteChannel::new; - - @Theory - public void mustCloseCursorOnClose( - ThrowingFunction channelConstructor ) throws Exception - { - AtomicInteger closeCounter = new AtomicInteger(); - PagedFile pf = new StubPagedFile( PageCache.PAGE_SIZE ) - { - @Override - public PageCursor io( long pageId, int pf_flags ) throws IOException - { - return new DelegatingPageCursor( super.io( pageId, pf_flags ) ) - { - @Override - public void close() - { - super.close(); - closeCounter.getAndIncrement(); - } - }; - } - }; - - channelConstructor.apply( pf ).close(); - assertThat( closeCounter.get(), is( 1 ) ); - } -} diff --git a/community/kernel/src/main/java/org/neo4j/kernel/NeoStoreDataSource.java b/community/kernel/src/main/java/org/neo4j/kernel/NeoStoreDataSource.java index e78096239757d..10f3cc3aaf3fa 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/NeoStoreDataSource.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/NeoStoreDataSource.java @@ -415,7 +415,7 @@ public void start() throws IOException LogVersionUpgradeChecker.check( tailScanner, config ); // Upgrade the store before we begin - RecordFormats formats = selectStoreFormats( config, storeDir, pageCache, logService ); + RecordFormats formats = selectStoreFormats( config, storeDir, fs, pageCache, logService ); upgradeStore( formats, tailScanner ); // Build all modules and their services @@ -540,11 +540,11 @@ public void start() throws IOException databaseHealth.healed(); } - private static RecordFormats selectStoreFormats( Config config, File storeDir, PageCache pageCache, + private static RecordFormats selectStoreFormats( Config config, File storeDir, FileSystemAbstraction fs, PageCache pageCache, LogService logService ) { LogProvider logging = logService.getInternalLogProvider(); - RecordFormats formats = RecordFormatSelector.selectNewestFormat( config, storeDir, pageCache, logging ); + RecordFormats formats = RecordFormatSelector.selectNewestFormat( config, storeDir, fs, pageCache, logging ); new RecordFormatPropertyConfigurator( formats, config ).configure(); return formats; } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/labelscan/GBPTreePageCacheFileUtil.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/labelscan/GBPTreePageCacheFileUtil.java deleted file mode 100644 index 146d33bfcd54a..0000000000000 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/labelscan/GBPTreePageCacheFileUtil.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Copyright (c) 2002-2018 "Neo Technology," - * Network Engine for Objects in Lund AB [http://neotechnology.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package org.neo4j.kernel.impl.index.labelscan; - -import java.io.File; -import java.io.IOException; -import java.nio.file.NoSuchFileException; - -import org.neo4j.io.fs.FileHandle; -import org.neo4j.io.pagecache.PageCache; -import org.neo4j.kernel.impl.index.GBPTreeFileUtil; - -import static org.neo4j.function.Predicates.alwaysTrue; - -/** - * Use {@link PageCache} to handle file operations on GBPTree file. - */ -public class GBPTreePageCacheFileUtil implements GBPTreeFileUtil -{ - private final PageCache pageCache; - - public GBPTreePageCacheFileUtil( PageCache pageCache ) - { - this.pageCache = pageCache; - } - - @Override - public void deleteFile( File storeFile ) throws IOException - { - FileHandle fileHandle = storeFileHandle( pageCache, storeFile ); - fileHandle.delete(); - } - - @Override - public void deleteFileIfPresent( File storeFile ) throws IOException - { - try - { - deleteFile( storeFile ); - } - catch ( NoSuchFileException e ) - { - // File does not exist, we don't need to delete - } - } - - @Override - public boolean storeFileExists( File storeFile ) - { - try - { - return pageCache.getCachedFileSystem().streamFilesRecursive( storeFile ).anyMatch( alwaysTrue() ); - } - catch ( IOException e ) - { - return false; - } - } - - @Override - public void mkdirs( File dir ) throws IOException - { - pageCache.getCachedFileSystem().mkdirs( dir ); - } - - private static FileHandle storeFileHandle( PageCache pageCache, File storeFile ) throws IOException - { - return pageCache.getCachedFileSystem() - .streamFilesRecursive( storeFile ) - .findFirst() - .orElseThrow( () -> new NoSuchFileException( storeFile.getPath() ) ); - } -} diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/labelscan/NativeLabelScanStore.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/labelscan/NativeLabelScanStore.java index 93a9bcf5b35a9..341fc70ffbf5b 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/labelscan/NativeLabelScanStore.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/labelscan/NativeLabelScanStore.java @@ -36,6 +36,7 @@ import org.neo4j.index.internal.gbptree.Layout; import org.neo4j.index.internal.gbptree.MetadataMismatchException; import org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector; +import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.IOLimiter; import org.neo4j.io.pagecache.PageCache; import org.neo4j.io.pagecache.PageCursor; @@ -44,6 +45,7 @@ import org.neo4j.kernel.api.labelscan.LabelScanWriter; import org.neo4j.kernel.impl.api.scan.FullStoreChangeStream; import org.neo4j.kernel.impl.index.GBPTreeFileUtil; +import org.neo4j.kernel.impl.index.schema.GBPTreeFileSystemFileUtil; import org.neo4j.kernel.impl.store.UnderlyingStorageException; import org.neo4j.kernel.monitoring.Monitors; import org.neo4j.storageengine.api.schema.LabelScanReader; @@ -168,17 +170,17 @@ public class NativeLabelScanStore implements LabelScanStore */ private static final Consumer writeClean = pageCursor -> pageCursor.putByte( CLEAN ); - public NativeLabelScanStore( PageCache pageCache, File storeDir, FullStoreChangeStream fullStoreChangeStream, + public NativeLabelScanStore( PageCache pageCache, File storeDir, FileSystemAbstraction fs, FullStoreChangeStream fullStoreChangeStream, boolean readOnly, Monitors monitors, RecoveryCleanupWorkCollector recoveryCleanupWorkCollector ) { - this( pageCache, storeDir, fullStoreChangeStream, readOnly, monitors, recoveryCleanupWorkCollector, + this( pageCache, storeDir, fs, fullStoreChangeStream, readOnly, monitors, recoveryCleanupWorkCollector, /*means no opinion about page size*/ 0 ); } /* * Test access to be able to control page size. */ - NativeLabelScanStore( PageCache pageCache, File storeDir, + NativeLabelScanStore( PageCache pageCache, File storeDir, FileSystemAbstraction fs, FullStoreChangeStream fullStoreChangeStream, boolean readOnly, Monitors monitors, RecoveryCleanupWorkCollector recoveryCleanupWorkCollector, int pageSize ) { @@ -191,7 +193,7 @@ public NativeLabelScanStore( PageCache pageCache, File storeDir, FullStoreChange this.monitors = monitors; this.monitor = monitors.newMonitor( Monitor.class ); this.recoveryCleanupWorkCollector = recoveryCleanupWorkCollector; - this.gbpTreeUtil = new GBPTreePageCacheFileUtil( pageCache ); + this.gbpTreeUtil = new GBPTreeFileSystemFileUtil( fs ); } /** diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NativeSchemaIndex.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NativeSchemaIndex.java index 3a9a3fa1a3bdf..cd8ebaded1253 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NativeSchemaIndex.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NativeSchemaIndex.java @@ -87,8 +87,6 @@ public void cleanupFinished( long numberOfPagesVisited, long numberOfCleanedCras private void ensureDirectoryExist() throws IOException { - // This will create the directory on the "normal" file system. - // When native index is put on blockdevice, page cache file system should be used instead. gbpTreeFileUtil.mkdirs( storeFile.getParentFile() ); } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/storageengine/impl/recordstorage/RecordStorageEngine.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/storageengine/impl/recordstorage/RecordStorageEngine.java index aaffc83dcdb8b..64e8af1d0174c 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/storageengine/impl/recordstorage/RecordStorageEngine.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/storageengine/impl/recordstorage/RecordStorageEngine.java @@ -209,7 +209,7 @@ public RecordStorageEngine( NeoStoreIndexStoreView neoStoreIndexStoreView = new NeoStoreIndexStoreView( lockService, neoStores ); Boolean readOnly = config.get( GraphDatabaseSettings.read_only ) && operationalMode == OperationalMode.single; monitors.addMonitorListener( new LoggingMonitor( logProvider.getLog( NativeLabelScanStore.class ) ) ); - labelScanStore = new NativeLabelScanStore( pageCache, storeDir, new FullLabelStream( neoStoreIndexStoreView ), + labelScanStore = new NativeLabelScanStore( pageCache, storeDir, fs, new FullLabelStream( neoStoreIndexStoreView ), readOnly, monitors, recoveryCleanupWorkCollector ); indexStoreView = new DynamicIndexStoreView( neoStoreIndexStoreView, labelScanStore, lockService, neoStores, logProvider ); diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/StoreFactory.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/StoreFactory.java index e60f00657caad..b5377d414c8a6 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/StoreFactory.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/StoreFactory.java @@ -76,7 +76,7 @@ public StoreFactory( File storeDir, Config config, IdGeneratorFactory idGenerato FileSystemAbstraction fileSystemAbstraction, LogProvider logProvider, VersionContextSupplier versionContextSupplier ) { this( storeDir, config, idGeneratorFactory, pageCache, fileSystemAbstraction, - selectForStoreOrConfig( config, storeDir, pageCache, logProvider ), + selectForStoreOrConfig( config, storeDir, fileSystemAbstraction, pageCache, logProvider ), logProvider, versionContextSupplier ); } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/StoreType.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/StoreType.java index 743fb9628eb39..926c07143e520 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/StoreType.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/StoreType.java @@ -23,7 +23,6 @@ import java.util.Optional; import org.neo4j.graphdb.factory.GraphDatabaseSettings; -import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.kernel.impl.index.labelscan.NativeLabelScanStore; import org.neo4j.kernel.impl.store.counts.CountsTracker; import org.neo4j.kernel.impl.store.id.IdType; @@ -255,7 +254,6 @@ public static Optional typeOf( String fileName ) /** * Returns whether or not store file by given file name should be managed by the page cache. - * Store files not managed by the page cache will end up on the otherwise specified {@link FileSystemAbstraction}. * * @param storeFileName file name of the store file to check. * @return Returns whether or not store file by given file name should be managed by the page cache. diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/format/RecordFormatSelector.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/format/RecordFormatSelector.java index c8083a9f9b78a..697bd3766cdc4 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/format/RecordFormatSelector.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/format/RecordFormatSelector.java @@ -31,6 +31,7 @@ import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.helpers.Service; +import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.store.MetaDataStore; @@ -132,17 +133,16 @@ public static RecordFormats selectForConfig( Config config, LogProvider logProvi * Note: package private only for testing. * * @param storeDir directory with the store + * @param fs file system used to access store files * @param pageCache page cache to read store files * @return record format of the given store or null if {@value MetaDataStore#DEFAULT_NAME} file not * found or can't be read */ @Nullable - static RecordFormats selectForStore( File storeDir, PageCache pageCache, LogProvider logProvider ) + static RecordFormats selectForStore( File storeDir, FileSystemAbstraction fs, PageCache pageCache, LogProvider logProvider ) { File neoStoreFile = new File( storeDir, MetaDataStore.DEFAULT_NAME ); - // It's important for the block device support, that we use the page cache file system to check for the - // neostore file. - if ( pageCache.getCachedFileSystem().fileExists( neoStoreFile ) ) + if ( fs.fileExists( neoStoreFile ) ) { try { @@ -175,18 +175,19 @@ static RecordFormats selectForStore( File storeDir, PageCache pageCache, LogProv * * @param config configuration parameters * @param storeDir directory with the store + * @param fs file system used to access store files * @param pageCache page cache to read store files * @return record format from the store (if it can be read) or configured record format or {@link #DEFAULT_FORMAT} * @throws IllegalArgumentException when configured format is different from the format present in the store */ @Nonnull public static RecordFormats selectForStoreOrConfig( - Config config, File storeDir, PageCache pageCache, LogProvider logProvider ) + Config config, File storeDir, FileSystemAbstraction fs, PageCache pageCache, LogProvider logProvider ) { RecordFormats configuredFormat = loadRecordFormat( configuredRecordFormat( config ) ); boolean formatConfigured = configuredFormat != null; - RecordFormats currentFormat = selectForStore( storeDir, pageCache, logProvider ); + RecordFormats currentFormat = selectForStore( storeDir, fs, pageCache, logProvider ); boolean storeWithFormatExists = currentFormat != null; if ( formatConfigured && storeWithFormatExists ) @@ -222,16 +223,17 @@ public static RecordFormats selectForStoreOrConfig( * exist yet - we consider formats as compatible. * @param config configuration parameters * @param storeDir directory with the store + * @param fs file system used to access store files * @param pageCache page cache to read store files * @param logProvider log provider * @return true if configured and actual format is compatible, false otherwise. */ public static boolean isStoreAndConfigFormatsCompatible( - Config config, File storeDir, PageCache pageCache, LogProvider logProvider ) + Config config, File storeDir, FileSystemAbstraction fs, PageCache pageCache, LogProvider logProvider ) { RecordFormats configuredFormat = loadRecordFormat( configuredRecordFormat( config ) ); - RecordFormats currentFormat = selectForStore( storeDir, pageCache, logProvider ); + RecordFormats currentFormat = selectForStore( storeDir, fs, pageCache, logProvider ); return (configuredFormat == null) || (currentFormat == null) || (currentFormat.getFormatFamily().equals( configuredFormat.getFormatFamily() ) && @@ -245,13 +247,14 @@ public static boolean isStoreAndConfigFormatsCompatible( * * @param config configuration parameters * @param storeDir directory with the store + * @param fs file system used to access store files * @param pageCache page cache to read store files * @return record format from the store (if it can be read) or configured record format or {@link #DEFAULT_FORMAT} * @see RecordFormats#generation() */ @Nonnull public static RecordFormats selectNewestFormat( - Config config, File storeDir, PageCache pageCache, LogProvider logProvider ) + Config config, File storeDir, FileSystemAbstraction fs, PageCache pageCache, LogProvider logProvider ) { boolean formatConfigured = StringUtils.isNotEmpty( configuredRecordFormat( config ) ); if ( formatConfigured ) @@ -261,7 +264,7 @@ public static RecordFormats selectNewestFormat( } else { - RecordFormats result = selectForStore( storeDir, pageCache, logProvider ); + RecordFormats result = selectForStore( storeDir, fs, pageCache, logProvider ); if ( result == null ) { // format was not explicitly configured and store does not exist, select default format diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/StoreUpgrader.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/StoreUpgrader.java index 1c6e229f3544f..7528073c459d6 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/StoreUpgrader.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/StoreUpgrader.java @@ -22,13 +22,11 @@ import java.io.File; import java.io.IOException; import java.io.UncheckedIOException; -import java.nio.file.NoSuchFileException; import java.util.ArrayList; import java.util.List; import java.util.regex.Pattern; import org.neo4j.graphdb.factory.GraphDatabaseSettings; -import org.neo4j.io.fs.FileHandle; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; import org.neo4j.kernel.configuration.Config; @@ -121,7 +119,7 @@ public void migrateIfNeeded( File storeDirectory ) { migrateStore( storeDirectory, migrationDirectory, migrationStateFile ); } - else if ( !RecordFormatSelector.isStoreAndConfigFormatsCompatible( config, storeDirectory, pageCache, logProvider ) ) + else if ( !RecordFormatSelector.isStoreAndConfigFormatsCompatible( config, storeDirectory, fileSystem, pageCache, logProvider ) ) { throw new UpgradeNotAllowedByConfigurationException(); } @@ -240,17 +238,6 @@ private void cleanMigrationDirectory( File migrationDirectory ) { fileSystem.deleteRecursively( migrationDirectory ); } - // We use the file system from the page cache here to make sure that the migration directory is clean - // even if we are using a block device. - try - { - pageCache.getCachedFileSystem().streamFilesRecursive( migrationDirectory ) - .forEach( FileHandle.HANDLE_DELETE ); - } - catch ( NoSuchFileException e ) - { - // This means that we had no files to clean, this is fine. - } } catch ( IOException | UncheckedIOException e ) { diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/participant/NativeLabelScanStoreMigrator.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/participant/NativeLabelScanStoreMigrator.java index 73eee3fba96fc..787fa6ccdc131 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/participant/NativeLabelScanStoreMigrator.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/participant/NativeLabelScanStoreMigrator.java @@ -100,8 +100,7 @@ public void moveMigratedFiles( File migrationDir, File storeDir, String versionT private void deleteNativeIndexFile( File storeDir ) throws IOException { - Optional indexFile = pageCache.getCachedFileSystem() - .streamFilesRecursive( NativeLabelScanStore.getLabelScanStoreFile( storeDir ) ).findFirst(); + Optional indexFile = fileSystem.streamFilesRecursive( NativeLabelScanStore.getLabelScanStoreFile( storeDir ) ).findFirst(); if ( indexFile.isPresent() ) { @@ -118,8 +117,7 @@ private void deleteNativeIndexFile( File storeDir ) throws IOException private void moveNativeIndexFile( File storeDir, File nativeLabelIndex ) throws IOException { - Optional nativeIndexFileHandle = - pageCache.getCachedFileSystem().streamFilesRecursive( nativeLabelIndex ).findFirst(); + Optional nativeIndexFileHandle = fileSystem.streamFilesRecursive( nativeLabelIndex ).findFirst(); if ( nativeIndexFileHandle.isPresent() ) { nativeIndexFileHandle.get().rename( new File( storeDir, NativeLabelScanStore.FILE_NAME ) ); @@ -130,7 +128,7 @@ private NativeLabelScanStore getNativeLabelScanStore( File migrationDir, ProgressReporter progressReporter, NeoStores neoStores ) { NeoStoreIndexStoreView neoStoreIndexStoreView = new NeoStoreIndexStoreView( NO_LOCK_SERVICE, neoStores ); - return new NativeLabelScanStore( pageCache, migrationDir, + return new NativeLabelScanStore( pageCache, migrationDir, fileSystem, new MonitoredFullLabelStream( neoStoreIndexStoreView, progressReporter ), false, new Monitors(), RecoveryCleanupWorkCollector.IMMEDIATE ); } @@ -146,8 +144,7 @@ private StoreFactory getStoreFactory( File storeDir, String versionToMigrateFrom private boolean isNativeLabelScanStoreMigrationRequired( File storeDir ) throws IOException { - return pageCache.getCachedFileSystem() - .streamFilesRecursive( new File( storeDir, NativeLabelScanStore.FILE_NAME ) ) + return fileSystem.streamFilesRecursive( new File( storeDir, NativeLabelScanStore.FILE_NAME ) ) .noneMatch( Predicates.alwaysTrue() ); } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigrator.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigrator.java index e1eaac3a96cad..14c1841f48945 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigrator.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigrator.java @@ -29,22 +29,14 @@ import java.io.OutputStream; import java.io.Writer; import java.nio.charset.StandardCharsets; -import java.nio.file.NoSuchFileException; -import java.nio.file.StandardCopyOption; -import java.nio.file.StandardOpenOption; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; -import java.util.Optional; -import java.util.function.Predicate; -import java.util.stream.StreamSupport; -import org.neo4j.io.fs.FileHandle; +import org.neo4j.helpers.collection.Iterables; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.fs.FileUtils; import org.neo4j.io.pagecache.PageCache; -import org.neo4j.io.pagecache.PageCursor; -import org.neo4j.io.pagecache.PagedFile; import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.logging.LogService; @@ -150,13 +142,12 @@ public void migrate( File storeDir, File migrationDir, ProgressReporter progress { if ( versionToMigrateFrom.equals( StandardV2_3.STORE_VERSION ) ) { - // These versions are not supported for block devices. CustomIOConfigValidator.assertCustomIOConfigNotUsed( config, CUSTOM_IO_EXCEPTION_MESSAGE ); } // Extract information about the last transaction from legacy neostore File neoStore = new File( storeDir, DEFAULT_NAME ); long lastTxId = MetaDataStore.getRecord( pageCache, neoStore, Position.LAST_TRANSACTION_ID ); - TransactionId lastTxInfo = extractTransactionIdInformation( neoStore, storeDir, lastTxId ); + TransactionId lastTxInfo = extractTransactionIdInformation( neoStore, lastTxId ); LogPosition lastTxLogPosition = extractTransactionLogPosition( neoStore, storeDir, lastTxId ); // Write the tx checksum to file in migrationDir, because we need it later when moving files into storeDir writeLastTxInformation( migrationDir, lastTxInfo ); @@ -269,7 +260,7 @@ private static File lastTxLogPositionFile( File migrationDir ) return new File( migrationDir, "lastxlogposition" ); } - TransactionId extractTransactionIdInformation( File neoStore, File storeDir, long lastTransactionId ) + TransactionId extractTransactionIdInformation( File neoStore, long lastTransactionId ) throws IOException { long checksum = MetaDataStore.getRecord( pageCache, neoStore, Position.LAST_TRANSACTION_CHECKSUM ); @@ -412,20 +403,6 @@ public boolean highIO() } StoreFile.fileOperation( DELETE, fileSystem, migrationDir, null, storesToDeleteFromMigratedDirectory, true, null, StoreFileType.values() ); - // When migrating on a block device there might be some files only accessible via the file system - // provided by the page cache. - try - { - Predicate fileHandlePredicate = fileHandle -> storesToDeleteFromMigratedDirectory.stream() - .anyMatch( storeFile -> storeFile.fileName( StoreFileType.STORE ) - .equals( fileHandle.getFile().getName() ) ); - pageCache.getCachedFileSystem().streamFilesRecursive( migrationDir ).filter( fileHandlePredicate ) - .forEach( FileHandle.HANDLE_DELETE ); - } - catch ( NoSuchFileException e ) - { - // This means that we had no files only present in the page cache, this is fine. - } } } @@ -462,25 +439,8 @@ private void prepareBatchImportMigration( File storeDir, File migrationDir, Reco StoreFile.NODE_LABEL_STORE}; if ( newFormat.dynamic().equals( oldFormat.dynamic() ) ) { - // We use the page cache for copying the STORE files since these might be on a block device. - for ( StoreFile file : storesFilesToMigrate ) - { - File fromPath = new File( storeDir, file.fileName( StoreFileType.STORE ) ); - File toPath = new File( migrationDir, file.fileName( StoreFileType.STORE ) ); - try - { - copyWithPageCache( fromPath, toPath ); - } - catch ( NoSuchFileException e ) - { - // It is okay for the file to not be there. - } - } - - // The ID files are to be kept on the normal file system, hence we use fileOperation to copy them. StoreFile.fileOperation( COPY, fileSystem, storeDir, migrationDir, Arrays.asList( storesFilesToMigrate ), - true, // OK if it's not there (1.9) - ExistingTargetStrategy.FAIL, StoreFileType.ID); + true, ExistingTargetStrategy.FAIL, StoreFileType.values()); } else { @@ -592,31 +552,6 @@ public void moveMigratedFiles( File migrationDir, File storeDir, String versionT true, // allow to skip non existent source files ExistingTargetStrategy.OVERWRITE, // allow to overwrite target files StoreFileType.values() ); - // Since some of the files might only be accessible through the file system provided by the page cache (i.e. - // block devices), we also try to move the files with the page cache. - try - { - Iterable fileHandles = pageCache.getCachedFileSystem() - .streamFilesRecursive( migrationDir )::iterator; - for ( FileHandle fh : fileHandles ) - { - Predicate predicate = - storeFile -> storeFile.fileName( StoreFileType.STORE ).equals( fh.getFile().getName() ); - if ( StreamSupport.stream( StoreFile.currentStoreFiles().spliterator(), false ).anyMatch( predicate ) ) - { - final Optional optionalPagedFile = pageCache.getExistingMapping( fh.getFile() ); - if ( optionalPagedFile.isPresent() ) - { - optionalPagedFile.get().close(); - } - fh.rename( new File( storeDir, fh.getFile().getName() ), StandardCopyOption.REPLACE_EXISTING ); - } - } - } - catch ( NoSuchFileException e ) - { - //This means that we had no files only present in the page cache, this is fine. - } } private void updateOrAddNeoStoreFieldsAsPartOfMigration( File migrationDir, File storeDir, @@ -624,7 +559,8 @@ private void updateOrAddNeoStoreFieldsAsPartOfMigration( File migrationDir, File { final File storeDirNeoStore = new File( storeDir, DEFAULT_NAME ); final File migrationDirNeoStore = new File( migrationDir, DEFAULT_NAME ); - copyWithPageCache( storeDirNeoStore, migrationDirNeoStore ); + StoreFile.fileOperation( COPY, fileSystem, storeDir, migrationDir, Iterables.iterable( StoreFile.NEO_STORE ), true, ExistingTargetStrategy.SKIP, + StoreFileType.STORE ); MetaDataStore.setRecord( pageCache, migrationDirNeoStore, Position.UPGRADE_TRANSACTION_ID, MetaDataStore.getRecord( pageCache, storeDirNeoStore, Position.LAST_TRANSACTION_ID ) ); @@ -677,27 +613,6 @@ public String toString() return "Kernel StoreMigrator"; } - private void copyWithPageCache( File sourceFile, File targetFile ) throws IOException - { - // We use the page cache for copying the neostore since it might be on a block device. - int pageSize = pageCache.pageSize(); - try ( PagedFile fromFile = pageCache.map( sourceFile, pageSize ); - PagedFile toFile = pageCache.map( targetFile, pageSize, StandardOpenOption.CREATE ); - PageCursor fromCursor = fromFile.io( 0L, PagedFile.PF_SHARED_READ_LOCK ); - PageCursor toCursor = toFile.io( 0L, PagedFile.PF_SHARED_WRITE_LOCK ) ) - { - while ( fromCursor.next() ) - { - toCursor.next(); - do - { - fromCursor.copyTo( 0, toCursor, 0, pageSize ); - } - while ( fromCursor.shouldRetry() ); - } - } - } - private static class NodeRecordChunk extends StoreScanChunk { NodeRecordChunk( RecordCursor recordCursor, NeoStores neoStores, boolean requiresPropertyMigration ) diff --git a/community/kernel/src/main/java/org/neo4j/kernel/internal/KernelDiagnostics.java b/community/kernel/src/main/java/org/neo4j/kernel/internal/KernelDiagnostics.java index 845422a87640c..8af7b44d7b3bc 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/internal/KernelDiagnostics.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/internal/KernelDiagnostics.java @@ -144,7 +144,7 @@ private static class MappedFileCounter { private long size; - public void addFile( File file ) + void addFile( File file ) { if ( StoreType.canBeManagedByPageCache( file.getName() ) ) { diff --git a/community/kernel/src/main/java/org/neo4j/unsafe/batchinsert/internal/BatchInserterImpl.java b/community/kernel/src/main/java/org/neo4j/unsafe/batchinsert/internal/BatchInserterImpl.java index 824f4dd4673a1..66ed24d60548f 100644 --- a/community/kernel/src/main/java/org/neo4j/unsafe/batchinsert/internal/BatchInserterImpl.java +++ b/community/kernel/src/main/java/org/neo4j/unsafe/batchinsert/internal/BatchInserterImpl.java @@ -253,7 +253,7 @@ public BatchInserterImpl( final File storeDir, final FileSystemAbstraction fileS this.idGeneratorFactory = new DefaultIdGeneratorFactory( fileSystem ); LogProvider internalLogProvider = logService.getInternalLogProvider(); - RecordFormats recordFormats = RecordFormatSelector.selectForStoreOrConfig( config, storeDir, + RecordFormats recordFormats = RecordFormatSelector.selectForStoreOrConfig( config, storeDir, fileSystem, pageCache, internalLogProvider ); StoreFactory sf = new StoreFactory( this.storeDir, config, idGeneratorFactory, pageCache, fileSystem, recordFormats, internalLogProvider, EmptyVersionContextSupplier.EMPTY ); @@ -301,7 +301,7 @@ public BatchInserterImpl( final File storeDir, final FileSystemAbstraction fileS IndexProvider provider = extensions.resolveDependency( IndexProvider.class, HighestSelectionStrategy.INSTANCE ); schemaIndexProviders = new DefaultIndexProviderMap( provider ); - labelScanStore = new NativeLabelScanStore( pageCache, storeDir, FullStoreChangeStream.EMPTY, false, new Monitors(), + labelScanStore = new NativeLabelScanStore( pageCache, storeDir, fileSystem, FullStoreChangeStream.EMPTY, false, new Monitors(), RecoveryCleanupWorkCollector.IMMEDIATE ); life.add( labelScanStore ); actions = new BatchSchemaActions(); diff --git a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/store/BatchingNeoStores.java b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/store/BatchingNeoStores.java index 0f785b957a046..09450466972ea 100644 --- a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/store/BatchingNeoStores.java +++ b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/store/BatchingNeoStores.java @@ -208,14 +208,13 @@ public void pruneAndOpenExistingStore( Predicate mainStoresToKeep, Pr private void deleteStoreFiles( String storeName, Predicate storesToKeep ) { - FileSystemAbstraction fs = pageCache.getCachedFileSystem(); for ( StoreType type : StoreType.values() ) { if ( type.isRecordStore() && !storesToKeep.test( type ) ) { for ( StoreFileType fileType : StoreFileType.values() ) { - fs.deleteFile( new File( storeDir, fileType.augment( storeName + type.getStoreFile().fileNamePart() ) ) ); + fileSystem.deleteFile( new File( storeDir, fileType.augment( storeName + type.getStoreFile().fileNamePart() ) ) ); } } } @@ -225,7 +224,7 @@ private void instantiateKernelExtensions() { life = new LifeSupport(); life.start(); - labelScanStore = new NativeLabelScanStore( pageCache, storeDir, FullStoreChangeStream.EMPTY, false, new Monitors(), + labelScanStore = new NativeLabelScanStore( pageCache, storeDir, fileSystem, FullStoreChangeStream.EMPTY, false, new Monitors(), RecoveryCleanupWorkCollector.IMMEDIATE ); life.add( labelScanStore ); } diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/index/AbstractGBPTreeFileUtilTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/index/AbstractGBPTreeFileUtilTest.java index c9d75e11dbaaa..1aac07bee69bc 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/index/AbstractGBPTreeFileUtilTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/index/AbstractGBPTreeFileUtilTest.java @@ -31,9 +31,6 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; -/** - * This test is inherited on blockdevice to test that behaviour is unified between product and blockdevice. - */ public abstract class AbstractGBPTreeFileUtilTest { private GBPTreeFileUtil fileUtil; diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/index/GBPTreeFileUtilTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/index/GBPTreeFileUtilTest.java index 7562ef6f13cfb..155ee791d1b5b 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/index/GBPTreeFileUtilTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/index/GBPTreeFileUtilTest.java @@ -30,7 +30,6 @@ import java.util.Collection; import org.neo4j.io.fs.FileSystemAbstraction; -import org.neo4j.kernel.impl.index.labelscan.GBPTreePageCacheFileUtil; import org.neo4j.kernel.impl.index.schema.GBPTreeFileSystemFileUtil; import org.neo4j.test.rule.PageCacheAndDependenciesRule; import org.neo4j.test.rule.TestDirectory; @@ -60,9 +59,7 @@ public static void extractFileSystem() @Parameterized.Parameters( name = "{0}" ) public static Collection fileUtils() { - return Arrays.asList( - new GBPTreePageCacheFileUtil( pageCacheAndDependenciesRule.pageCache() ), - new GBPTreeFileSystemFileUtil( pageCacheAndDependenciesRule.fileSystem() ) ); + return Arrays.asList( new GBPTreeFileSystemFileUtil( pageCacheAndDependenciesRule.fileSystem() ) ); } @Parameterized.Parameter diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/index/labelscan/NativeLabelScanStoreIT.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/index/labelscan/NativeLabelScanStoreIT.java index 9b4b76b1077fa..d605a2bebba92 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/index/labelscan/NativeLabelScanStoreIT.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/index/labelscan/NativeLabelScanStoreIT.java @@ -62,8 +62,9 @@ public class NativeLabelScanStoreIT @Before public void before() { - PageCache pageCache = pageCacheRule.getPageCache( new DefaultFileSystemAbstraction() ); - store = life.add( new NativeLabelScanStore( pageCache, directory.absolutePath(), FullStoreChangeStream.EMPTY, + DefaultFileSystemAbstraction fileSystem = new DefaultFileSystemAbstraction(); + PageCache pageCache = pageCacheRule.getPageCache( fileSystem ); + store = life.add( new NativeLabelScanStore( pageCache, directory.absolutePath(), fileSystem, FullStoreChangeStream.EMPTY, false, new Monitors(), RecoveryCleanupWorkCollector.IMMEDIATE, // a bit of random pageSize Math.min( pageCache.pageSize(), 256 << random.nextInt( 5 ) ) ) ); diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/index/labelscan/NativeLabelScanStoreRebuildTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/index/labelscan/NativeLabelScanStoreRebuildTest.java index 149b290459dbc..a16676ed05e67 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/index/labelscan/NativeLabelScanStoreRebuildTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/index/labelscan/NativeLabelScanStoreRebuildTest.java @@ -84,7 +84,7 @@ public void mustBeDirtyIfFailedDuringRebuild() throws Exception monitors.addMonitorListener( monitor ); NativeLabelScanStore nativeLabelScanStore = - new NativeLabelScanStore( pageCache, storeDir, EMPTY, false, monitors, IMMEDIATE ); + new NativeLabelScanStore( pageCache, storeDir, fileSystemRule.get(), EMPTY, false, monitors, IMMEDIATE ); nativeLabelScanStore.init(); nativeLabelScanStore.start(); @@ -106,7 +106,7 @@ public void doNotRebuildIfOpenedInReadOnlyModeAndIndexIsNotClean() throws IOExce monitors.addMonitorListener( monitor ); NativeLabelScanStore nativeLabelScanStore = - new NativeLabelScanStore( pageCache, storeDir, EMPTY, true, monitors, RecoveryCleanupWorkCollector.IGNORE ); + new NativeLabelScanStore( pageCache, storeDir, fileSystemRule.get(), EMPTY, true, monitors, RecoveryCleanupWorkCollector.IGNORE ); nativeLabelScanStore.init(); nativeLabelScanStore.start(); @@ -127,7 +127,7 @@ public void labelScanStoreIsDirtyWhenIndexIsNotClean() throws IOException monitors.addMonitorListener( monitor ); NativeLabelScanStore nativeLabelScanStore = - new NativeLabelScanStore( pageCache, storeDir, EMPTY, true, monitors, RecoveryCleanupWorkCollector.IGNORE ); + new NativeLabelScanStore( pageCache, storeDir, fileSystemRule.get(), EMPTY, true, monitors, RecoveryCleanupWorkCollector.IGNORE ); nativeLabelScanStore.init(); nativeLabelScanStore.start(); @@ -147,7 +147,7 @@ public void shouldFailOnUnsortedLabelsFromFullStoreChangeStream() throws Excepti try { nativeLabelScanStore = - new NativeLabelScanStore( pageCache, storeDir, changeStream, false, new Monitors(), IMMEDIATE ); + new NativeLabelScanStore( pageCache, storeDir, fileSystemRule.get(), changeStream, false, new Monitors(), IMMEDIATE ); nativeLabelScanStore.init(); // when @@ -174,7 +174,7 @@ private void createDirtyIndex( PageCache pageCache ) throws IOException NativeLabelScanStore nativeLabelScanStore = null; try { - nativeLabelScanStore = new NativeLabelScanStore( pageCache, storeDir, THROWING_STREAM, false, + nativeLabelScanStore = new NativeLabelScanStore( pageCache, storeDir, fileSystemRule.get(), THROWING_STREAM, false, new Monitors(), IMMEDIATE ); nativeLabelScanStore.init(); diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/index/labelscan/NativeLabelScanStoreTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/index/labelscan/NativeLabelScanStoreTest.java index 6331c4177dca3..b35fc26c05cb5 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/index/labelscan/NativeLabelScanStoreTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/index/labelscan/NativeLabelScanStoreTest.java @@ -65,7 +65,7 @@ private LabelScanStore getLabelScanStore( FileSystemAbstraction fileSystemAbstra FullStoreChangeStream fullStoreChangeStream, boolean readOnly, Monitors monitors ) { PageCache pageCache = pageCacheRule.getPageCache( fileSystemAbstraction ); - return new NativeLabelScanStore( pageCache, rootFolder, + return new NativeLabelScanStore( pageCache, rootFolder, fileSystemAbstraction, fullStoreChangeStream, readOnly, monitors, RecoveryCleanupWorkCollector.IMMEDIATE ); } diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/StoreFactoryTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/StoreFactoryTest.java index 7197ce26c7781..cc71e9d48307e 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/StoreFactoryTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/StoreFactoryTest.java @@ -73,7 +73,7 @@ public void setUp() private StoreFactory storeFactory( Config config, OpenOption... openOptions ) { LogProvider logProvider = NullLogProvider.getInstance(); - RecordFormats recordFormats = selectForStoreOrConfig( config, storeDir, pageCache, logProvider ); + RecordFormats recordFormats = selectForStoreOrConfig( config, storeDir, fsRule, pageCache, logProvider ); return new StoreFactory( storeDir, DEFAULT_NAME, config, idGeneratorFactory, pageCache, fsRule.get(), recordFormats, logProvider, EmptyVersionContextSupplier.EMPTY, openOptions ); } diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/participant/NativeLabelScanStoreMigratorTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/participant/NativeLabelScanStoreMigratorTest.java index 1dad867df7296..e1989cf2a983a 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/participant/NativeLabelScanStoreMigratorTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/participant/NativeLabelScanStoreMigratorTest.java @@ -201,7 +201,7 @@ public void reportProgressOnNativeIndexPopulation() throws IOException private NativeLabelScanStore getNativeLabelScanStore( File dir, boolean readOnly ) { - return new NativeLabelScanStore( pageCache, dir, FullStoreChangeStream.EMPTY, readOnly, new Monitors(), + return new NativeLabelScanStore( pageCache, dir, fileSystem, FullStoreChangeStream.EMPTY, readOnly, new Monitors(), RecoveryCleanupWorkCollector.IGNORE ); } diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigratorTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigratorTest.java index 4f6f1d13776a1..eb5376e2daa58 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigratorTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigratorTest.java @@ -110,7 +110,7 @@ public void shouldExtractTransactionInformationFromMetaDataStore() throws Except // ... and with migrator StoreMigrator migrator = new StoreMigrator( fileSystemRule.get(), pageCache, config, logService ); - TransactionId actual = migrator.extractTransactionIdInformation( neoStore, storeDir, txId ); + TransactionId actual = migrator.extractTransactionIdInformation( neoStore, txId ); // then assertEquals( expected, actual ); @@ -134,7 +134,7 @@ public void shouldGenerateTransactionInformationWhenLogsNotPresent() throws Exce assertEquals( FIELD_NOT_PRESENT, getRecord( pageCache, neoStore, LAST_TRANSACTION_COMMIT_TIMESTAMP ) ); // ... and with migrator StoreMigrator migrator = new StoreMigrator( fileSystemRule.get(), pageCache, config, logService ); - TransactionId actual = migrator.extractTransactionIdInformation( neoStore, storeDir, txId ); + TransactionId actual = migrator.extractTransactionIdInformation( neoStore, txId ); // then assertEquals( txId, actual.transactionId() ); @@ -205,7 +205,7 @@ public void shouldGenerateTransactionInformationWhenLogsAreEmpty() throws Except assertEquals( FIELD_NOT_PRESENT, getRecord( pageCache, neoStore, LAST_TRANSACTION_COMMIT_TIMESTAMP ) ); // ... and with migrator StoreMigrator migrator = new StoreMigrator( fileSystemRule.get(), pageCache, config, logService ); - TransactionId actual = migrator.extractTransactionIdInformation( neoStore, storeDir, txId ); + TransactionId actual = migrator.extractTransactionIdInformation( neoStore, txId ); // then assertEquals( txId, actual.transactionId() ); diff --git a/community/kernel/src/test/java/org/neo4j/unsafe/batchinsert/internal/BatchInsertTest.java b/community/kernel/src/test/java/org/neo4j/unsafe/batchinsert/internal/BatchInsertTest.java index 14215f0c6e996..cabe54ca73e16 100644 --- a/community/kernel/src/test/java/org/neo4j/unsafe/batchinsert/internal/BatchInsertTest.java +++ b/community/kernel/src/test/java/org/neo4j/unsafe/batchinsert/internal/BatchInsertTest.java @@ -60,10 +60,11 @@ import org.neo4j.helpers.collection.MapUtil; import org.neo4j.helpers.collection.Pair; import org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector; +import org.neo4j.io.fs.DefaultFileSystemAbstraction; import org.neo4j.kernel.api.exceptions.index.IndexEntryConflictException; import org.neo4j.kernel.api.index.IndexPopulator; -import org.neo4j.kernel.api.index.PropertyAccessor; import org.neo4j.kernel.api.index.IndexProvider; +import org.neo4j.kernel.api.index.PropertyAccessor; import org.neo4j.kernel.api.labelscan.LabelScanStore; import org.neo4j.kernel.api.schema.index.SchemaIndexDescriptor; import org.neo4j.kernel.api.schema.index.SchemaIndexDescriptorFactory; @@ -1414,7 +1415,8 @@ public void shouldIgnoreRemovingNonExistentRelationshipProperty() private LabelScanStore getLabelScanStore() { - return new NativeLabelScanStore( pageCacheRule.getPageCache( fileSystemRule.get() ), storeDir.absolutePath(), + DefaultFileSystemAbstraction fs = fileSystemRule.get(); + return new NativeLabelScanStore( pageCacheRule.getPageCache( fs ), storeDir.absolutePath(), fs, FullStoreChangeStream.EMPTY, true, new Monitors(), RecoveryCleanupWorkCollector.IMMEDIATE ); } diff --git a/enterprise/backup/src/main/java/org/neo4j/backup/OnlineBackupKernelExtension.java b/enterprise/backup/src/main/java/org/neo4j/backup/OnlineBackupKernelExtension.java index bc91f4b47face..7239405587243 100644 --- a/enterprise/backup/src/main/java/org/neo4j/backup/OnlineBackupKernelExtension.java +++ b/enterprise/backup/src/main/java/org/neo4j/backup/OnlineBackupKernelExtension.java @@ -92,7 +92,7 @@ public OnlineBackupKernelExtension( Config config, final GraphDatabaseAPI graphD TransactionIdStore transactionIdStore = transactionIdStoreSupplier.get(); StoreCopyServer copier = new StoreCopyServer( neoStoreDataSource, checkPointerSupplier.get(), fileSystemAbstraction, graphDatabaseAPI.getStoreDir(), - monitors.newMonitor( StoreCopyServer.Monitor.class ), pageCache, storeCopyCheckPointMutex ); + monitors.newMonitor( StoreCopyServer.Monitor.class ), storeCopyCheckPointMutex ); LogicalTransactionStore logicalTransactionStore = logicalTransactionStoreSupplier.get(); LogFileInformation logFileInformation = logFileInformationSupplier.get(); return new BackupImpl( copier, logicalTransactionStore, transactionIdStore, logFileInformation, diff --git a/enterprise/backup/src/main/java/org/neo4j/backup/impl/BackupCopyService.java b/enterprise/backup/src/main/java/org/neo4j/backup/impl/BackupCopyService.java index fc00753e24811..162e47813efd0 100644 --- a/enterprise/backup/src/main/java/org/neo4j/backup/impl/BackupCopyService.java +++ b/enterprise/backup/src/main/java/org/neo4j/backup/impl/BackupCopyService.java @@ -103,7 +103,7 @@ public void clearIdFiles( Path backupLocation ) throws IOException boolean backupExists( Path destination ) { - File[] files = pageCache.getCachedFileSystem().listFiles( destination.toFile() ); + File[] files = fs.listFiles( destination.toFile() ); return files != null && Arrays.stream( files ).anyMatch( f -> f.isFile() && f.getName().endsWith( MetaDataStore.DEFAULT_NAME ) ); } diff --git a/enterprise/backup/src/main/java/org/neo4j/backup/impl/BackupStrategyCoordinatorFactory.java b/enterprise/backup/src/main/java/org/neo4j/backup/impl/BackupStrategyCoordinatorFactory.java index 0eaf8fcdce6d1..25aeaca15aeec 100644 --- a/enterprise/backup/src/main/java/org/neo4j/backup/impl/BackupStrategyCoordinatorFactory.java +++ b/enterprise/backup/src/main/java/org/neo4j/backup/impl/BackupStrategyCoordinatorFactory.java @@ -65,7 +65,7 @@ BackupStrategyCoordinator backupStrategyCoordinator( BackupDelegator backupDelegator, PageCache pageCache ) { FileSystemAbstraction fs = outsideWorld.fileSystem(); - BackupCopyService copyService = new BackupCopyService( fs, pageCache, new FileMoveProvider( pageCache, fs ) ); + BackupCopyService copyService = new BackupCopyService( fs, pageCache, new FileMoveProvider( fs ) ); ProgressMonitorFactory progressMonitorFactory = ProgressMonitorFactory.textual( outsideWorld.errorStream() ); BackupRecoveryService recoveryService = new BackupRecoveryService(); long timeout = onlineBackupContext.getRequiredArguments().getTimeout(); diff --git a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/RegularCatchupServerHandler.java b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/RegularCatchupServerHandler.java index 5d09f6fa8de31..e5156577873d5 100644 --- a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/RegularCatchupServerHandler.java +++ b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/RegularCatchupServerHandler.java @@ -35,7 +35,6 @@ import org.neo4j.causalclustering.core.state.snapshot.CoreSnapshotRequestHandler; import org.neo4j.causalclustering.identity.StoreId; import org.neo4j.io.fs.FileSystemAbstraction; -import org.neo4j.io.pagecache.PageCache; import org.neo4j.kernel.NeoStoreDataSource; import org.neo4j.kernel.impl.transaction.log.LogicalTransactionStore; import org.neo4j.kernel.impl.transaction.log.TransactionIdStore; @@ -55,17 +54,15 @@ public class RegularCatchupServerHandler implements CatchupServerHandler private final Supplier dataSourceSupplier; private final BooleanSupplier dataSourceAvailabilitySupplier; private final FileSystemAbstraction fs; - private final PageCache pageCache; private final StoreCopyCheckPointMutex storeCopyCheckPointMutex; private final CoreSnapshotService snapshotService; private final Supplier checkPointerSupplier; public RegularCatchupServerHandler( Monitors monitors, LogProvider logProvider, Supplier storeIdSupplier, Supplier transactionIdStoreSupplier, Supplier logicalTransactionStoreSupplier, - Supplier dataSourceSupplier, BooleanSupplier dataSourceAvailabilitySupplier, FileSystemAbstraction fs, PageCache pageCache, + Supplier dataSourceSupplier, BooleanSupplier dataSourceAvailabilitySupplier, FileSystemAbstraction fs, StoreCopyCheckPointMutex storeCopyCheckPointMutex, CoreSnapshotService snapshotService, Supplier checkPointerSupplier ) { - this.monitors = monitors; this.logProvider = logProvider; this.storeIdSupplier = storeIdSupplier; @@ -74,7 +71,6 @@ public RegularCatchupServerHandler( Monitors monitors, LogProvider logProvider, this.dataSourceSupplier = dataSourceSupplier; this.dataSourceAvailabilitySupplier = dataSourceAvailabilitySupplier; this.fs = fs; - this.pageCache = pageCache; this.storeCopyCheckPointMutex = storeCopyCheckPointMutex; this.snapshotService = snapshotService; this.checkPointerSupplier = checkPointerSupplier; @@ -97,22 +93,21 @@ public ChannelHandler getStoreIdRequestHandler( CatchupServerProtocol catchupSer public ChannelHandler storeListingRequestHandler( CatchupServerProtocol catchupServerProtocol ) { return new PrepareStoreCopyRequestHandler( catchupServerProtocol, checkPointerSupplier, storeCopyCheckPointMutex, dataSourceSupplier, - new PrepareStoreCopyFilesProvider( pageCache, fs ) ); + new PrepareStoreCopyFilesProvider( fs ) ); } @Override public ChannelHandler getStoreFileRequestHandler( CatchupServerProtocol catchupServerProtocol ) { return new StoreCopyRequestHandler.GetStoreFileRequestHandler( catchupServerProtocol, dataSourceSupplier, checkPointerSupplier, - new StoreFileStreamingProtocol(), pageCache, fs, - logProvider ); + new StoreFileStreamingProtocol(), fs, logProvider ); } @Override public ChannelHandler getIndexSnapshotRequestHandler( CatchupServerProtocol catchupServerProtocol ) { return new StoreCopyRequestHandler.GetIndexSnapshotRequestHandler( catchupServerProtocol, dataSourceSupplier, checkPointerSupplier, - new StoreFileStreamingProtocol(), pageCache, fs, logProvider ); + new StoreFileStreamingProtocol(), fs, logProvider ); } @Override diff --git a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/PrepareStoreCopyFiles.java b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/PrepareStoreCopyFiles.java index debfd02d38147..4255299e62a49 100644 --- a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/PrepareStoreCopyFiles.java +++ b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/PrepareStoreCopyFiles.java @@ -30,7 +30,6 @@ import org.neo4j.graphdb.ResourceIterator; import org.neo4j.io.fs.FileSystemAbstraction; -import org.neo4j.io.pagecache.PageCache; import org.neo4j.kernel.NeoStoreDataSource; import org.neo4j.kernel.impl.store.StoreType; import org.neo4j.storageengine.api.StoreFileMetadata; @@ -40,14 +39,12 @@ public class PrepareStoreCopyFiles implements AutoCloseable { private final NeoStoreDataSource neoStoreDataSource; - private final PageCache pageCache; private final FileSystemAbstraction fileSystemAbstraction; private final CloseablesListener closeablesListener = new CloseablesListener(); - PrepareStoreCopyFiles( NeoStoreDataSource neoStoreDataSource, PageCache pageCache, FileSystemAbstraction fileSystemAbstraction ) + PrepareStoreCopyFiles( NeoStoreDataSource neoStoreDataSource, FileSystemAbstraction fileSystemAbstraction ) { this.neoStoreDataSource = neoStoreDataSource; - this.pageCache = pageCache; this.fileSystemAbstraction = fileSystemAbstraction; } @@ -108,7 +105,7 @@ private StoreResource toStoreResource( StoreFileMetadata storeFileMetadata ) thr File storeDir = neoStoreDataSource.getStoreDir(); File file = storeFileMetadata.file(); String relativePath = relativePath( storeDir, file ); - return new StoreResource( file, relativePath, storeFileMetadata.recordSize(), pageCache, fileSystemAbstraction ); + return new StoreResource( file, relativePath, storeFileMetadata.recordSize(), fileSystemAbstraction ); } @Override diff --git a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/PrepareStoreCopyFilesProvider.java b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/PrepareStoreCopyFilesProvider.java index 69dee758b69b0..7259bc9bf9654 100644 --- a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/PrepareStoreCopyFilesProvider.java +++ b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/PrepareStoreCopyFilesProvider.java @@ -19,25 +19,20 @@ */ package org.neo4j.causalclustering.catchup.storecopy; -import java.util.function.Supplier; - import org.neo4j.io.fs.FileSystemAbstraction; -import org.neo4j.io.pagecache.PageCache; import org.neo4j.kernel.NeoStoreDataSource; public class PrepareStoreCopyFilesProvider { - private final PageCache pageCache; private final FileSystemAbstraction fileSystemAbstraction; - public PrepareStoreCopyFilesProvider( PageCache pageCache, FileSystemAbstraction fileSystemAbstraction ) + public PrepareStoreCopyFilesProvider( FileSystemAbstraction fileSystemAbstraction ) { - this.pageCache = pageCache; this.fileSystemAbstraction = fileSystemAbstraction; } PrepareStoreCopyFiles prepareStoreCopyFiles( NeoStoreDataSource neoStoreDataSource ) { - return new PrepareStoreCopyFiles( neoStoreDataSource, pageCache, fileSystemAbstraction ); + return new PrepareStoreCopyFiles( neoStoreDataSource, fileSystemAbstraction ); } } diff --git a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/RemoteStore.java b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/RemoteStore.java index 32b362c6edef1..c2989e55b0c92 100644 --- a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/RemoteStore.java +++ b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/RemoteStore.java @@ -125,7 +125,7 @@ public void copy( CatchupAddressProvider addressProvider, StoreId expectedStoreI try { long lastFlushedTxId; - StreamToDiskProvider streamToDiskProvider = new StreamToDiskProvider( destDir, fs, pageCache, monitors ); + StreamToDiskProvider streamToDiskProvider = new StreamToDiskProvider( destDir, fs, monitors ); lastFlushedTxId = storeCopyClient.copyStoreFiles( addressProvider, expectedStoreId, streamToDiskProvider, () -> new MaximumTotalTime( config.get( CausalClusteringSettings.store_copy_max_retry_time_per_request ).getSeconds(), TimeUnit.SECONDS ), destDir ); diff --git a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/StoreCopyRequestHandler.java b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/StoreCopyRequestHandler.java index 0aa5073c00294..6fb003f04f361 100644 --- a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/StoreCopyRequestHandler.java +++ b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/StoreCopyRequestHandler.java @@ -35,7 +35,6 @@ import org.neo4j.graphdb.ResourceIterator; import org.neo4j.helpers.collection.Iterators; import org.neo4j.io.fs.FileSystemAbstraction; -import org.neo4j.io.pagecache.PageCache; import org.neo4j.kernel.NeoStoreDataSource; import org.neo4j.kernel.impl.transaction.log.checkpoint.CheckPointer; import org.neo4j.logging.Log; @@ -53,18 +52,17 @@ public abstract class StoreCopyRequestHandler extend private final Supplier dataSource; private final Supplier checkpointerSupplier; private final StoreFileStreamingProtocol storeFileStreamingProtocol; - private final PageCache pageCache; + private final FileSystemAbstraction fs; private final Log log; StoreCopyRequestHandler( CatchupServerProtocol protocol, Supplier dataSource, Supplier checkpointerSupplier, - StoreFileStreamingProtocol storeFileStreamingProtocol, PageCache pageCache, FileSystemAbstraction fs, LogProvider logProvider ) + StoreFileStreamingProtocol storeFileStreamingProtocol, FileSystemAbstraction fs, LogProvider logProvider ) { this.protocol = protocol; this.dataSource = dataSource; this.checkpointerSupplier = checkpointerSupplier; this.storeFileStreamingProtocol = storeFileStreamingProtocol; - this.pageCache = pageCache; this.fs = fs; this.log = logProvider.getLog( StoreCopyRequestHandler.class ); } @@ -95,7 +93,7 @@ else if ( !isTransactionWithinReach( request.requiredTransactionId(), checkpoint StoreFileMetadata storeFileMetadata = resourceIterator.next(); storeFileStreamingProtocol.stream( ctx, new StoreResource( storeFileMetadata.file(), relativePath( storeDir, storeFileMetadata.file() ), storeFileMetadata.recordSize(), - pageCache, fs ) ); + fs ) ); } } responseStatus = StoreCopyFinishedResponse.Status.SUCCESS; @@ -127,9 +125,9 @@ private static Predicate matchesRequested( String fileName ) public static class GetStoreFileRequestHandler extends StoreCopyRequestHandler { public GetStoreFileRequestHandler( CatchupServerProtocol protocol, Supplier dataSource, Supplier checkpointerSupplier, - StoreFileStreamingProtocol storeFileStreamingProtocol, PageCache pageCache, FileSystemAbstraction fs, LogProvider logProvider ) + StoreFileStreamingProtocol storeFileStreamingProtocol, FileSystemAbstraction fs, LogProvider logProvider ) { - super( protocol, dataSource, checkpointerSupplier, storeFileStreamingProtocol, pageCache, fs, logProvider ); + super( protocol, dataSource, checkpointerSupplier, storeFileStreamingProtocol, fs, logProvider ); } @Override @@ -147,10 +145,10 @@ ResourceIterator files( GetStoreFileRequest request, NeoStore public static class GetIndexSnapshotRequestHandler extends StoreCopyRequestHandler { public GetIndexSnapshotRequestHandler( CatchupServerProtocol protocol, Supplier dataSource, - Supplier checkpointerSupplier, StoreFileStreamingProtocol storeFileStreamingProtocol, PageCache pageCache, + Supplier checkpointerSupplier, StoreFileStreamingProtocol storeFileStreamingProtocol, FileSystemAbstraction fs, LogProvider logProvider ) { - super( protocol, dataSource, checkpointerSupplier, storeFileStreamingProtocol, pageCache, fs, logProvider ); + super( protocol, dataSource, checkpointerSupplier, storeFileStreamingProtocol, fs, logProvider ); } @Override diff --git a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/StoreFiles.java b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/StoreFiles.java index d380f6dc00b73..023eea8e58efb 100644 --- a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/StoreFiles.java +++ b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/StoreFiles.java @@ -93,7 +93,7 @@ private Stream acceptedPageCachedFiles( File storeDir ) throws IOExc { try { - Stream stream = pageCache.getCachedFileSystem().streamFilesRecursive( storeDir ); + Stream stream = fs.streamFilesRecursive( storeDir ); Predicate acceptableFiles = fh -> fileFilter.accept( storeDir, fh.getRelativeFile().getPath() ); return stream.filter( acceptableFiles ); } diff --git a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/StoreResource.java b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/StoreResource.java index d0561a47c364d..e151c02fc386d 100644 --- a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/StoreResource.java +++ b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/StoreResource.java @@ -23,44 +23,27 @@ import java.io.IOException; import java.nio.channels.ReadableByteChannel; import java.util.Objects; -import java.util.Optional; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.fs.OpenMode; -import org.neo4j.io.pagecache.PageCache; -import org.neo4j.io.pagecache.PagedFile; class StoreResource { private final File file; private final String path; private final int recordSize; - private final PageCache pageCache; private final FileSystemAbstraction fs; - StoreResource( File file, String relativePath, int recordSize, PageCache pageCache, FileSystemAbstraction fs ) + StoreResource( File file, String relativePath, int recordSize, FileSystemAbstraction fs ) { this.file = file; this.path = relativePath; this.recordSize = recordSize; - this.pageCache = pageCache; this.fs = fs; } ReadableByteChannel open() throws IOException { - if ( !pageCache.fileSystemSupportsFileOperations() ) - { - Optional existingMapping = pageCache.getExistingMapping( file ); - if ( existingMapping.isPresent() ) - { - try ( PagedFile pagedFile = existingMapping.get() ) - { - return pagedFile.openReadableByteChannel(); - } - } - } - return fs.open( file, OpenMode.READ ); } diff --git a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/StoreResourceStreamFactory.java b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/StoreResourceStreamFactory.java index 1372196c80d3f..16d7ba37b4c7d 100644 --- a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/StoreResourceStreamFactory.java +++ b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/StoreResourceStreamFactory.java @@ -26,7 +26,6 @@ import org.neo4j.cursor.RawCursor; import org.neo4j.graphdb.ResourceIterator; import org.neo4j.io.fs.FileSystemAbstraction; -import org.neo4j.io.pagecache.PageCache; import org.neo4j.kernel.NeoStoreDataSource; import org.neo4j.storageengine.api.StoreFileMetadata; @@ -34,13 +33,11 @@ public class StoreResourceStreamFactory { - private final PageCache pageCache; private final FileSystemAbstraction fs; private final Supplier dataSourceSupplier; - public StoreResourceStreamFactory( PageCache pageCache, FileSystemAbstraction fs, Supplier dataSourceSupplier ) + public StoreResourceStreamFactory( FileSystemAbstraction fs, Supplier dataSourceSupplier ) { - this.pageCache = pageCache; this.fs = fs; this.dataSourceSupplier = dataSourceSupplier; } @@ -73,7 +70,7 @@ public boolean next() throws IOException StoreFileMetadata md = files.next(); - resource = new StoreResource( md.file(), relativePath( storeDir, md.file() ), md.recordSize(), pageCache, fs ); + resource = new StoreResource( md.file(), relativePath( storeDir, md.file() ), md.recordSize(), fs ); return true; } diff --git a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/StreamToDisk.java b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/StreamToDisk.java index a9a7d2feee96a..bcebc8a35ce9a 100644 --- a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/StreamToDisk.java +++ b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/StreamToDisk.java @@ -29,7 +29,6 @@ import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.fs.OpenMode; -import org.neo4j.io.pagecache.PagedFile; import static org.neo4j.io.IOUtils.closeAll; @@ -38,11 +37,6 @@ public class StreamToDisk implements StoreFileStream private WritableByteChannel writableByteChannel; private List closeables; - static StreamToDisk fromPagedFile( PagedFile pagedFile ) throws IOException - { - return new StreamToDisk( pagedFile.openWritableByteChannel(), pagedFile ); - } - static StreamToDisk fromFile( FileSystemAbstraction fsa, File file ) throws IOException { return new StreamToDisk( fsa.open( file, OpenMode.READ_WRITE ) ); diff --git a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/StreamToDiskProvider.java b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/StreamToDiskProvider.java index 56afbb81b3ca7..f3231ac07250d 100644 --- a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/StreamToDiskProvider.java +++ b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/StreamToDiskProvider.java @@ -21,27 +21,21 @@ import java.io.File; import java.io.IOException; -import java.nio.file.StandardOpenOption; import org.neo4j.causalclustering.catchup.tx.FileCopyMonitor; import org.neo4j.io.fs.FileSystemAbstraction; -import org.neo4j.io.pagecache.PageCache; -import org.neo4j.io.pagecache.PagedFile; -import org.neo4j.kernel.impl.store.StoreType; import org.neo4j.kernel.monitoring.Monitors; public class StreamToDiskProvider implements StoreFileStreamProvider { private final File storeDir; private final FileSystemAbstraction fs; - private final PageCache pageCache; private final FileCopyMonitor fileCopyMonitor; - StreamToDiskProvider( File storeDir, FileSystemAbstraction fs, PageCache pageCache, Monitors monitors ) + StreamToDiskProvider( File storeDir, FileSystemAbstraction fs, Monitors monitors ) { this.storeDir = storeDir; this.fs = fs; - this.pageCache = pageCache; this.fileCopyMonitor = monitors.newMonitor( FileCopyMonitor.class ); } @@ -51,15 +45,6 @@ public StoreFileStream acquire( String destination, int requiredAlignment ) thro File fileName = new File( storeDir, destination ); fs.mkdirs( fileName.getParentFile() ); fileCopyMonitor.copyFile( fileName ); - if ( !pageCache.fileSystemSupportsFileOperations() && StoreType.canBeManagedByPageCache( destination ) ) - { - int filePageSize = pageCache.pageSize() - pageCache.pageSize() % requiredAlignment; - PagedFile pagedFile = pageCache.map( fileName, filePageSize, StandardOpenOption.CREATE ); - return StreamToDisk.fromPagedFile( pagedFile ); - } - else - { - return StreamToDisk.fromFile( fs, fileName ); - } + return StreamToDisk.fromFile( fs, fileName ); } } diff --git a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/core/server/CoreServerModule.java b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/core/server/CoreServerModule.java index db0be393374db..60befc1fc39d6 100644 --- a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/core/server/CoreServerModule.java +++ b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/core/server/CoreServerModule.java @@ -65,8 +65,8 @@ import org.neo4j.causalclustering.core.state.storage.StateStorage; import org.neo4j.causalclustering.helper.CompositeSuspendable; import org.neo4j.causalclustering.helper.ExponentialBackoffStrategy; -import org.neo4j.causalclustering.messaging.LifecycleMessageHandler; import org.neo4j.causalclustering.helper.Suspendable; +import org.neo4j.causalclustering.messaging.LifecycleMessageHandler; import org.neo4j.causalclustering.net.InstalledProtocolHandler; import org.neo4j.causalclustering.net.Server; import org.neo4j.causalclustering.protocol.ModifierProtocolInstaller; @@ -192,7 +192,7 @@ public CoreServerModule( IdentityModule identityModule, final PlatformModule pla CatchupServerHandler catchupServerHandler = new RegularCatchupServerHandler( platformModule.monitors, logProvider, localDatabase::storeId, platformModule.dependencies.provideDependency( TransactionIdStore.class ), platformModule.dependencies.provideDependency( LogicalTransactionStore.class ), localDatabase::dataSource, localDatabase::isAvailable, - fileSystem, platformModule.pageCache, platformModule.storeCopyCheckPointMutex, snapshotService, + fileSystem, platformModule.storeCopyCheckPointMutex, snapshotService, new CheckpointerSupplier( platformModule.dependencies ) ); CatchupProtocolServerInstaller.Factory catchupProtocolServerInstaller = new CatchupProtocolServerInstaller.Factory( serverPipelineBuilderFactory, diff --git a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/readreplica/EnterpriseReadReplicaEditionModule.java b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/readreplica/EnterpriseReadReplicaEditionModule.java index de3a533e65769..8a4f32fe606ff 100644 --- a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/readreplica/EnterpriseReadReplicaEditionModule.java +++ b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/readreplica/EnterpriseReadReplicaEditionModule.java @@ -315,7 +315,7 @@ public EnterpriseReadReplicaEditionModule( final PlatformModule platformModule, RegularCatchupServerHandler catchupServerHandler = new RegularCatchupServerHandler( platformModule.monitors, logProvider, localDatabase::storeId, platformModule.dependencies.provideDependency( TransactionIdStore.class ), platformModule.dependencies.provideDependency( LogicalTransactionStore.class ), localDatabase::dataSource, localDatabase::isAvailable, - fileSystem, platformModule.pageCache, platformModule.storeCopyCheckPointMutex, null, new CheckpointerSupplier( platformModule.dependencies ) ); + fileSystem, platformModule.storeCopyCheckPointMutex, null, new CheckpointerSupplier( platformModule.dependencies ) ); InstalledProtocolHandler installedProtocolHandler = new InstalledProtocolHandler(); // TODO: hook into a procedure Server catchupServer = new CatchupServerBuilder( catchupServerHandler ) diff --git a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/storecopy/FakeCatchupServer.java b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/storecopy/FakeCatchupServer.java index a9966c6f224e7..9f7433c4ed676 100644 --- a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/storecopy/FakeCatchupServer.java +++ b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/storecopy/FakeCatchupServer.java @@ -40,13 +40,10 @@ import org.neo4j.causalclustering.catchup.ResponseMessageType; import org.neo4j.causalclustering.identity.StoreId; import org.neo4j.io.fs.FileSystemAbstraction; -import org.neo4j.io.pagecache.PageCache; import org.neo4j.logging.Log; import org.neo4j.logging.LogProvider; import org.neo4j.test.rule.TestDirectory; -import static org.mockito.Mockito.mock; - class TestCatchupServerHandler implements CatchupServerHandler { private final Set filesystem = new HashSet<>(); @@ -151,7 +148,7 @@ private void incrementRequestCount( File file ) private StoreResource storeResourceFromEntry( File file ) { file = testDirectory.file( file.getName() ); - return new StoreResource( file, file.getAbsolutePath(), 16, mock( PageCache.class ), fileSystemAbstraction ); + return new StoreResource( file, file.getAbsolutePath(), 16, fileSystemAbstraction ); } @Override diff --git a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/storecopy/FileSenderTest.java b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/storecopy/FileSenderTest.java index 3ff97683e1dcb..349bdeeee3af2 100644 --- a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/storecopy/FileSenderTest.java +++ b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/storecopy/FileSenderTest.java @@ -72,7 +72,7 @@ public void sendEmptyFile() throws Exception // given File emptyFile = testDirectory.file( "emptyFile" ); fs.create( emptyFile ).close(); - FileSender fileSender = new FileSender( new StoreResource( emptyFile, null, 16, pageCache, fs ) ); + FileSender fileSender = new FileSender( new StoreResource( emptyFile, null, 16, fs ) ); // when + then assertFalse( fileSender.isEndOfInput() ); @@ -94,7 +94,7 @@ public void sendSmallFile() throws Exception storeChannel.write( ByteBuffer.wrap( bytes ) ); } - FileSender fileSender = new FileSender( new StoreResource( smallFile, null, 16, pageCache, fs ) ); + FileSender fileSender = new FileSender( new StoreResource( smallFile, null, 16, fs ) ); // when + then assertFalse( fileSender.isEndOfInput() ); @@ -117,7 +117,7 @@ public void sendLargeFile() throws Exception storeChannel.write( ByteBuffer.wrap( bytes ) ); } - FileSender fileSender = new FileSender( new StoreResource( smallFile, null, 16, pageCache, fs ) ); + FileSender fileSender = new FileSender( new StoreResource( smallFile, null, 16, fs ) ); // when + then assertFalse( fileSender.isEndOfInput() ); @@ -140,7 +140,7 @@ public void sendLargeFileWithSizeMultipleOfTheChunkSize() throws Exception storeChannel.write( ByteBuffer.wrap( bytes ) ); } - FileSender fileSender = new FileSender( new StoreResource( smallFile, null, 16, pageCache, fs ) ); + FileSender fileSender = new FileSender( new StoreResource( smallFile, null, 16, fs ) ); // when + then assertFalse( fileSender.isEndOfInput() ); @@ -157,7 +157,7 @@ public void sendEmptyFileWhichGrowsBeforeSendCommences() throws Exception // given File file = testDirectory.file( "file" ); StoreChannel writer = fs.create( file ); - FileSender fileSender = new FileSender( new StoreResource( file, null, 16, pageCache, fs ) ); + FileSender fileSender = new FileSender( new StoreResource( file, null, 16, fs ) ); // when byte[] bytes = writeRandomBytes( writer, 1024 ); @@ -175,7 +175,7 @@ public void sendEmptyFileWhichGrowsWithPartialChunkSizes() throws Exception // given File file = testDirectory.file( "file" ); StoreChannel writer = fs.create( file ); - FileSender fileSender = new FileSender( new StoreResource( file, null, 16, pageCache, fs ) ); + FileSender fileSender = new FileSender( new StoreResource( file, null, 16, fs ) ); // when byte[] chunkA = writeRandomBytes( writer, MAX_SIZE ); @@ -200,7 +200,7 @@ public void sendFileWhichGrowsAfterLastChunkWasSent() throws Exception // given File file = testDirectory.file( "file" ); StoreChannel writer = fs.create( file ); - FileSender fileSender = new FileSender( new StoreResource( file, null, 16, pageCache, fs ) ); + FileSender fileSender = new FileSender( new StoreResource( file, null, 16, fs ) ); // when byte[] chunkA = writeRandomBytes( writer, MAX_SIZE ); @@ -224,7 +224,7 @@ public void sendLargerFileWhichGrows() throws Exception // given File file = testDirectory.file( "file" ); StoreChannel writer = fs.create( file ); - FileSender fileSender = new FileSender( new StoreResource( file, null, 16, pageCache, fs ) ); + FileSender fileSender = new FileSender( new StoreResource( file, null, 16, fs ) ); // when byte[] chunkA = writeRandomBytes( writer, MAX_SIZE ); @@ -267,7 +267,7 @@ public void sendLargeFileWithUnreliableReadBufferSize() throws Exception Adversary adversary = new RandomAdversary( 0.9, 0.0, 0.0 ); AdversarialFileSystemAbstraction afs = new AdversarialFileSystemAbstraction( adversary, fs ); - FileSender fileSender = new FileSender( new StoreResource( smallFile, null, 16, pageCache, afs ) ); + FileSender fileSender = new FileSender( new StoreResource( smallFile, null, 16, afs ) ); // when + then assertFalse( fileSender.isEndOfInput() ); diff --git a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/storecopy/PrepareStoreCopyFilesTest.java b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/storecopy/PrepareStoreCopyFilesTest.java index 15940894768bf..e7898198acbb2 100644 --- a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/storecopy/PrepareStoreCopyFilesTest.java +++ b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/storecopy/PrepareStoreCopyFilesTest.java @@ -33,8 +33,6 @@ import org.neo4j.helpers.collection.Iterators; import org.neo4j.io.fs.DefaultFileSystemAbstraction; import org.neo4j.io.fs.FileSystemAbstraction; -import org.neo4j.io.pagecache.PageCache; -import org.neo4j.io.pagecache.impl.muninn.StandalonePageCacheFactory; import org.neo4j.kernel.NeoStoreDataSource; import org.neo4j.kernel.impl.transaction.state.NeoStoreFileIndexListing; import org.neo4j.kernel.impl.transaction.state.NeoStoreFileListing; @@ -57,7 +55,6 @@ public class PrepareStoreCopyFilesTest @Rule public final TestDirectory testDirectory = TestDirectory.testDirectory( fileSystemAbstraction ); private PrepareStoreCopyFiles prepareStoreCopyFiles; - private PageCache pageCache; private NeoStoreFileIndexListing indexListingMock; private File storeDir; private NeoStoreFileListing.StoreFileListingBuilder fileListingBuilder; @@ -65,7 +62,6 @@ public class PrepareStoreCopyFilesTest @Before public void setUp() { - pageCache = StandalonePageCacheFactory.createPageCache( fileSystemAbstraction ); NeoStoreDataSource dataSource = mock( NeoStoreDataSource.class ); fileListingBuilder = mock( NeoStoreFileListing.StoreFileListingBuilder.class, CALLS_REAL_METHODS ); storeDir = testDirectory.graphDbDir(); @@ -76,7 +72,7 @@ public void setUp() when( storeFileListing.getNeoStoreFileIndexListing() ).thenReturn( indexListingMock ); when( storeFileListing.builder() ).thenReturn( fileListingBuilder ); when( dataSource.getNeoStoreFileListing() ).thenReturn( storeFileListing ); - prepareStoreCopyFiles = new PrepareStoreCopyFiles( dataSource, pageCache, fileSystemAbstraction ); + prepareStoreCopyFiles = new PrepareStoreCopyFiles( dataSource, fileSystemAbstraction ); } @Test @@ -109,7 +105,7 @@ public void shouldReturnExpectedListOfFileNamesForEachType() throws Exception //then File[] expectedFilesConverted = Arrays.stream( expectedFiles ).map( StoreFileMetadata::file ).toArray( File[]::new ); StoreResource[] exeptedAtomicFilesConverted = Arrays.stream( expectedFiles ).map( - f -> new StoreResource( f.file(), getRelativePath( f ), f.recordSize(), pageCache, fileSystemAbstraction ) ).toArray( StoreResource[]::new ); + f -> new StoreResource( f.file(), getRelativePath( f ), f.recordSize(), fileSystemAbstraction ) ).toArray( StoreResource[]::new ); assertArrayEquals( expectedFilesConverted, files ); assertEquals( exeptedAtomicFilesConverted.length, atomicFilesSnapshot.length ); for ( int i = 0; i < exeptedAtomicFilesConverted.length; i++ ) diff --git a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/storecopy/SimpleCatchupClient.java b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/storecopy/SimpleCatchupClient.java index 78906957e3844..83a26be8c1e44 100644 --- a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/storecopy/SimpleCatchupClient.java +++ b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/storecopy/SimpleCatchupClient.java @@ -60,7 +60,7 @@ class SimpleCatchupClient implements AutoCloseable from = getCatchupServerAddress(); correctStoreId = getStoreIdFromKernelStoreId( graphDb ); clientPageCache = createPageCache(); - streamToDiskProvider = new StreamToDiskProvider( temporaryDirectory, fsa, clientPageCache, new Monitors() ); + streamToDiskProvider = new StreamToDiskProvider( temporaryDirectory, fsa, new Monitors() ); log = logProvider.getLog( SimpleCatchupClient.class ); this.logProvider = logProvider; } diff --git a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/storecopy/StoreCopyClientIT.java b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/storecopy/StoreCopyClientIT.java index 0da7877460928..ee5e143aa89cb 100644 --- a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/storecopy/StoreCopyClientIT.java +++ b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/storecopy/StoreCopyClientIT.java @@ -34,9 +34,6 @@ import java.io.Reader; import java.nio.ByteBuffer; import java.nio.CharBuffer; -import java.nio.channels.ReadableByteChannel; -import java.nio.channels.WritableByteChannel; -import java.nio.file.StandardOpenOption; import java.util.Arrays; import java.util.HashSet; import java.util.Iterator; @@ -58,9 +55,6 @@ import org.neo4j.io.fs.DefaultFileSystemAbstraction; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.fs.StoreChannel; -import org.neo4j.io.pagecache.PageCache; -import org.neo4j.io.pagecache.PagedFile; -import org.neo4j.io.pagecache.impl.muninn.StandalonePageCacheFactory; import org.neo4j.kernel.monitoring.Monitors; import org.neo4j.logging.FormattedLogProvider; import org.neo4j.logging.Level; @@ -69,8 +63,6 @@ import org.neo4j.test.rule.TestDirectory; import static org.junit.Assert.assertEquals; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.when; public class StoreCopyClientIT { @@ -184,7 +176,7 @@ public void shouldNotAppendToFileWhenRetryingWithNewFile() throws Throwable { // given String fileName = "foo"; - String pageCacheFileName = "bar"; + String copyFileName = "bar"; String unfinishedContent = "abcd"; String finishedContent = "abcdefgh"; Iterator contents = Iterators.iterator( unfinishedContent, finishedContent ); @@ -198,33 +190,28 @@ public ChannelHandler getStoreFileRequestHandler( CatchupServerProtocol catchupS return new SimpleChannelInboundHandler() { @Override - protected void channelRead0( ChannelHandlerContext ctx, GetStoreFileRequest msg ) throws IOException + protected void channelRead0( ChannelHandlerContext ctx, GetStoreFileRequest msg ) { // create the files and write the given content File file = new File( fileName ); + File fileCopy = new File( copyFileName ); String thisConent = contents.next(); writeContents( fsa, file, thisConent ); - PageCache pageCache = neverSupportingFileOperationPageCache( StandalonePageCacheFactory.createPageCache( fsa ) ); - PagedFile pagedFile = - pageCache.map( new File( pageCacheFileName ), pageCache.pageSize(), StandardOpenOption.CREATE, StandardOpenOption.WRITE ); - try ( WritableByteChannel writableByteChannel = pagedFile.openWritableByteChannel() ) - { - writableByteChannel.write( ByteBuffer.wrap( thisConent.getBytes( Charsets.UTF_8 ) ) ); - } - - sendFile( ctx, file, pageCache ); - sendFile( ctx, pagedFile.file(), pageCache ); + writeContents( fsa, fileCopy, thisConent ); + + sendFile( ctx, file ); + sendFile( ctx, fileCopy ); StoreCopyFinishedResponse.Status status = contents.hasNext() ? StoreCopyFinishedResponse.Status.E_UNKNOWN : StoreCopyFinishedResponse.Status.SUCCESS; new StoreFileStreamingProtocol().end( ctx, status ); catchupServerProtocol.expect( CatchupServerProtocol.State.MESSAGE_TYPE ); } - private void sendFile( ChannelHandlerContext ctx, File file, PageCache pageCache ) + private void sendFile( ChannelHandlerContext ctx, File file ) { ctx.write( ResponseMessageType.FILE ); ctx.write( new FileHeader( file.getName() ) ); - ctx.writeAndFlush( new FileSender( new StoreResource( file, file.getName(), 16, pageCache, fsa ) ) ).addListener( + ctx.writeAndFlush( new FileSender( new StoreResource( file, file.getName(), 16, fsa ) ) ).addListener( future -> fsa.deleteFile( file ) ); } }; @@ -273,8 +260,7 @@ protected void channelRead0( ChannelHandlerContext ctx, GetIndexFilesRequest msg StoreId storeId = halfWayFailingServerhandler.getStoreId(); File storeDir = testDirectory.makeGraphDbDir(); - PageCache pageCache = StandalonePageCacheFactory.createPageCache( fsa ); - StreamToDiskProvider streamToDiskProvider = new StreamToDiskProvider( storeDir, fsa, pageCache, new Monitors() ); + StreamToDiskProvider streamToDiskProvider = new StreamToDiskProvider( storeDir, fsa, new Monitors() ); // and subject.copyStoreFiles( addressProvider, storeId, streamToDiskProvider, () -> defaultTerminationCondition, targetLocation ); @@ -283,11 +269,12 @@ protected void channelRead0( ChannelHandlerContext ctx, GetIndexFilesRequest msg assertEquals( fileContent( new File( storeDir, fileName ) ), finishedContent ); // and - PagedFile pagedFile = pageCache.map( new File( storeDir, pageCacheFileName ), pageCache.pageSize(), StandardOpenOption.READ ); + File fileCopy = new File( storeDir, copyFileName ); + ByteBuffer buffer = ByteBuffer.wrap( new byte[finishedContent.length()] ); - try ( ReadableByteChannel readableByteChannel = pagedFile.openReadableByteChannel() ) + try ( StoreChannel storeChannel = fsa.create( fileCopy ) ) { - readableByteChannel.read( buffer ); + storeChannel.read( buffer ); } assertEquals( finishedContent, new String( buffer.array(), Charsets.UTF_8 ) ); } @@ -298,13 +285,6 @@ protected void channelRead0( ChannelHandlerContext ctx, GetIndexFilesRequest msg } } - private PageCache neverSupportingFileOperationPageCache( PageCache pageCache ) - { - PageCache spy = spy( pageCache ); - when( spy.fileSystemSupportsFileOperations() ).thenReturn( false ); - return spy; - } - private static AdvertisedSocketAddress from( int port ) { return new AdvertisedSocketAddress( "localhost", port ); diff --git a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/storecopy/StoreCopyRequestHandlerTest.java b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/storecopy/StoreCopyRequestHandlerTest.java index 8342d41bb5e0e..5b9dabd15ca6e 100644 --- a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/storecopy/StoreCopyRequestHandlerTest.java +++ b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/storecopy/StoreCopyRequestHandlerTest.java @@ -24,7 +24,6 @@ import org.junit.Test; import java.io.File; -import java.io.IOException; import java.util.function.Supplier; import org.neo4j.causalclustering.catchup.CatchupServerProtocol; @@ -35,7 +34,6 @@ import org.neo4j.helpers.collection.Iterators; import org.neo4j.io.fs.DefaultFileSystemAbstraction; import org.neo4j.io.fs.FileSystemAbstraction; -import org.neo4j.io.pagecache.PageCache; import org.neo4j.kernel.NeoStoreDataSource; import org.neo4j.kernel.impl.transaction.log.checkpoint.CheckPointer; import org.neo4j.kernel.impl.transaction.log.checkpoint.TriggerInfo; @@ -57,7 +55,6 @@ public class StoreCopyRequestHandlerTest private final NeoStoreDataSource neoStoreDataSource = mock( NeoStoreDataSource.class ); private final CheckPointer checkPointer = new FakeCheckPointer(); - private final PageCache pageCache = mock( PageCache.class ); private EmbeddedChannel embeddedChannel; private CatchupServerProtocol catchupServerProtocol; @@ -68,7 +65,7 @@ public void setup() catchupServerProtocol.expect( CatchupServerProtocol.State.GET_STORE_FILE ); StoreCopyRequestHandler storeCopyRequestHandler = new NiceStoreCopyRequestHandler( catchupServerProtocol, () -> neoStoreDataSource, () -> checkPointer, new StoreFileStreamingProtocol(), - pageCache, fileSystemAbstraction, NullLogProvider.getInstance() ); + fileSystemAbstraction, NullLogProvider.getInstance() ); when( neoStoreDataSource.getStoreId() ).thenReturn( new org.neo4j.kernel.impl.store.StoreId( 1, 2, 5, 3, 4 ) ); embeddedChannel = new EmbeddedChannel( storeCopyRequestHandler ); } @@ -119,11 +116,11 @@ public void shouldResetProtocolAndGiveErrorOnUncheckedException() } @Test - public void shoulResetProtoclAndGiveErrorIfFilesThrowException() + public void shoulResetProtocolAndGiveErrorIfFilesThrowException() { EmbeddedChannel alternativeChannel = new EmbeddedChannel( new EvilStoreCopyRequestHandler( catchupServerProtocol, () -> neoStoreDataSource, () -> checkPointer, new StoreFileStreamingProtocol(), - pageCache, fileSystemAbstraction, NullLogProvider.getInstance() ) ); + fileSystemAbstraction, NullLogProvider.getInstance() ) ); try { alternativeChannel.writeInbound( new GetStoreFileRequest( STORE_ID_MATCHING, new File( "some-file" ), 1 ) ); @@ -143,14 +140,14 @@ public void shoulResetProtoclAndGiveErrorIfFilesThrowException() private class NiceStoreCopyRequestHandler extends StoreCopyRequestHandler { private NiceStoreCopyRequestHandler( CatchupServerProtocol protocol, Supplier dataSource, - Supplier checkpointerSupplier, StoreFileStreamingProtocol storeFileStreamingProtocol, PageCache pageCache, + Supplier checkpointerSupplier, StoreFileStreamingProtocol storeFileStreamingProtocol, FileSystemAbstraction fs, LogProvider logProvider ) { - super( protocol, dataSource, checkpointerSupplier, storeFileStreamingProtocol, pageCache, fs, logProvider ); + super( protocol, dataSource, checkpointerSupplier, storeFileStreamingProtocol, fs, logProvider ); } @Override - ResourceIterator files( StoreCopyRequest request, NeoStoreDataSource neoStoreDataSource ) throws IOException + ResourceIterator files( StoreCopyRequest request, NeoStoreDataSource neoStoreDataSource ) { return Iterators.emptyResourceIterator(); } @@ -159,14 +156,14 @@ ResourceIterator files( StoreCopyRequest request, NeoStoreDat private class EvilStoreCopyRequestHandler extends StoreCopyRequestHandler { private EvilStoreCopyRequestHandler( CatchupServerProtocol protocol, Supplier dataSource, - Supplier checkpointerSupplier, StoreFileStreamingProtocol storeFileStreamingProtocol, PageCache pageCache, + Supplier checkpointerSupplier, StoreFileStreamingProtocol storeFileStreamingProtocol, FileSystemAbstraction fs, LogProvider logProvider ) { - super( protocol, dataSource, checkpointerSupplier, storeFileStreamingProtocol, pageCache, fs, logProvider ); + super( protocol, dataSource, checkpointerSupplier, storeFileStreamingProtocol, fs, logProvider ); } @Override - ResourceIterator files( StoreCopyRequest request, NeoStoreDataSource neoStoreDataSource ) throws IOException + ResourceIterator files( StoreCopyRequest request, NeoStoreDataSource neoStoreDataSource ) { throw new IllegalStateException( "I am evil" ); } diff --git a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/storecopy/StoreFileStreamingProtocolTest.java b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/storecopy/StoreFileStreamingProtocolTest.java index fad11a26d0f26..9a9a06e9e81b1 100644 --- a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/storecopy/StoreFileStreamingProtocolTest.java +++ b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/storecopy/StoreFileStreamingProtocolTest.java @@ -20,7 +20,6 @@ package org.neo4j.causalclustering.catchup.storecopy; import io.netty.channel.ChannelHandlerContext; -import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.mockito.InOrder; @@ -34,7 +33,6 @@ import org.neo4j.causalclustering.catchup.ResponseMessageType; import org.neo4j.cursor.RawCursor; -import org.neo4j.io.pagecache.PageCache; import org.neo4j.test.rule.PageCacheRule; import org.neo4j.test.rule.fs.EphemeralFileSystemRule; @@ -52,14 +50,6 @@ public class StoreFileStreamingProtocolTest @Rule public PageCacheRule pageCacheRule = new PageCacheRule(); - private PageCache pageCache; - - @Before - public void setup() - { - pageCache = pageCacheRule.getPageCache( fs.get() ); - } - @Test public void shouldStreamResources() throws Exception { @@ -134,6 +124,6 @@ public void shouldBeAbleToEndWithSuccess() private StoreResource createResource( File file, int recordSize ) throws IOException { fs.create( file ); - return new StoreResource( file, file.getPath(), recordSize, pageCache, fs ); + return new StoreResource( file, file.getPath(), recordSize, fs ); } } diff --git a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/storecopy/StoreFilesTest.java b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/storecopy/StoreFilesTest.java index 573dd75379385..99088f5558f33 100644 --- a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/storecopy/StoreFilesTest.java +++ b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/storecopy/StoreFilesTest.java @@ -27,14 +27,12 @@ import java.io.File; import java.io.FilenameFilter; import java.io.IOException; -import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.concurrent.ThreadLocalRandom; import java.util.function.Supplier; import org.neo4j.causalclustering.identity.StoreId; -import org.neo4j.graphdb.mockfs.EphemeralFileSystemAbstraction; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.fs.OpenMode; import org.neo4j.io.pagecache.PageCache; @@ -46,27 +44,23 @@ import org.neo4j.test.rule.TestDirectory; import org.neo4j.test.rule.fs.EphemeralFileSystemRule; +import static java.util.Arrays.asList; +import static java.util.Collections.singletonList; import static org.hamcrest.Matchers.is; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; -/** - * To support block device storage for causal clustering, it is important that the interaction with files go through - * either the normal file system, and/or through the page cache, depending on the file. - */ public class StoreFilesTest { protected TestDirectory testDirectory; protected Supplier fileSystemRule; - protected EphemeralFileSystemRule hiddenFileSystemRule; protected PageCacheRule pageCacheRule; @Rule public RuleChain rules; private FileSystemAbstraction fs; - private EphemeralFileSystemAbstraction pc; private PageCache pageCache; private StoreFiles storeFiles; private LogFiles logFiles; @@ -81,11 +75,9 @@ protected void createRules() testDirectory = TestDirectory.testDirectory( StoreFilesTest.class ); EphemeralFileSystemRule ephemeralFileSystemRule = new EphemeralFileSystemRule(); fileSystemRule = ephemeralFileSystemRule; - hiddenFileSystemRule = new EphemeralFileSystemRule(); pageCacheRule = new PageCacheRule(); rules = RuleChain.outerRule( ephemeralFileSystemRule ) .around( testDirectory ) - .around( hiddenFileSystemRule ) .around( pageCacheRule ); } @@ -93,8 +85,7 @@ protected void createRules() public void setUp() throws Exception { fs = fileSystemRule.get(); - pc = hiddenFileSystemRule.get(); - pageCache = pageCacheRule.getPageCache( pc ); + pageCache = pageCacheRule.getPageCache( fs ); storeFiles = new StoreFiles( fs, pageCache ); logFiles = LogFilesBuilder.logFilesBasedOnlyBuilder( testDirectory.directory(), fs ).build(); } @@ -104,11 +95,6 @@ private void createOnFileSystem( File file ) throws IOException createFile( fs, file ); } - private void createOnPageCache( File file ) throws IOException - { - createFile( hiddenFileSystemRule.get(), file ); - } - private void createFile( FileSystemAbstraction fs, File file ) throws IOException { fs.mkdirs( file.getParentFile() ); @@ -128,14 +114,12 @@ public void deleteMustRecursivelyRemoveFilesInGivenDirectory() throws Exception File b = new File( dir, "b" ); createOnFileSystem( a ); - createOnPageCache( b ); assertTrue( fs.fileExists( a ) ); - assertTrue( pc.fileExists( b ) ); + assertFalse( fs.fileExists( b ) ); storeFiles.delete( dir, logFiles ); assertFalse( fs.fileExists( a ) ); - assertFalse( pc.fileExists( b ) ); } @Test @@ -143,23 +127,17 @@ public void deleteMustNotDeleteIgnoredFiles() throws Exception { File dir = getBaseDir(); File a = new File( dir, "a" ); - File b = new File( dir, "b" ); File c = new File( dir, "c" ); - File d = new File( dir, "d" ); createOnFileSystem( a ); createOnFileSystem( c ); - createOnPageCache( b ); - createOnPageCache( d ); FilenameFilter filter = ( directory, name ) -> !name.equals( "c" ) && !name.equals( "d" ); storeFiles = new StoreFiles( fs, pageCache, filter ); storeFiles.delete( dir, logFiles ); assertFalse( fs.fileExists( a ) ); - assertFalse( pc.fileExists( b ) ); assertTrue( fs.fileExists( c ) ); - assertTrue( pc.fileExists( d ) ); } @Test @@ -168,23 +146,17 @@ public void deleteMustNotDeleteFilesInIgnoredDirectories() throws Exception File dir = getBaseDir(); File ignore = new File( dir, "ignore" ); File a = new File( dir, "a" ); - File b = new File( dir, "b" ); File c = new File( ignore, "c" ); - File d = new File( ignore, "d" ); createOnFileSystem( a ); createOnFileSystem( c ); - createOnPageCache( b ); - createOnPageCache( d ); FilenameFilter filter = ( directory, name ) -> !name.startsWith( "ignore" ); storeFiles = new StoreFiles( fs, pageCache, filter ); storeFiles.delete( dir, logFiles ); assertFalse( fs.fileExists( a ) ); - assertFalse( pc.fileExists( b ) ); assertTrue( fs.fileExists( c ) ); - assertTrue( pc.fileExists( d ) ); } @Test @@ -203,21 +175,16 @@ public void mustMoveFilesToTargetDirectory() throws Exception File src = new File( base, "src" ); File tgt = new File( base, "tgt" ); File a = new File( src, "a" ); - File b = new File( src, "b" ); createOnFileSystem( a ); - createOnPageCache( b ); // Ensure the 'tgt' directory exists createOnFileSystem( new File( tgt, ".fs-ignore" ) ); - createOnPageCache( new File( tgt, ".pc-ignore" ) ); storeFiles.moveTo( src, tgt, logFiles ); assertFalse( fs.fileExists( a ) ); - assertFalse( pc.fileExists( b ) ); assertTrue( fs.fileExists( new File( tgt, "a" ) ) ); - assertTrue( pc.fileExists( new File( tgt, "b" ) ) ); } @Test @@ -228,21 +195,16 @@ public void movedFilesMustRetainTheirRelativePaths() throws Exception File tgt = new File( base, "tgt" ); File dir = new File( src, "dir" ); File a = new File( dir, "a" ); - File b = new File( dir, "b" ); createOnFileSystem( a ); - createOnPageCache( b ); // Ensure the 'tgt' directory exists createOnFileSystem( new File( tgt, ".fs-ignore" ) ); - createOnPageCache( new File( tgt, ".pc-ignore" ) ); storeFiles.moveTo( src, tgt, logFiles ); assertFalse( fs.fileExists( a ) ); - assertFalse( pc.fileExists( b ) ); assertTrue( fs.fileExists( new File( new File( tgt, "dir" ), "a" ) ) ); - assertTrue( pc.fileExists( new File( new File( tgt, "dir" ), "b" ) ) ); } @Test @@ -251,31 +213,23 @@ public void moveMustIgnoreFilesFilteredOut() throws Exception File base = getBaseDir(); File src = new File( base, "src" ); File a = new File( src, "a" ); - File b = new File( src, "b" ); File ignore = new File( src, "ignore" ); File c = new File( ignore, "c" ); - File d = new File( ignore, "d" ); File tgt = new File( base, "tgt" ); createOnFileSystem( a ); - createOnPageCache( b ); createOnFileSystem( c ); - createOnPageCache( d ); // Ensure the 'tgt' directory exists createOnFileSystem( new File( tgt, ".fs-ignore" ) ); - createOnPageCache( new File( tgt, ".pc-ignore" ) ); FilenameFilter filter = ( directory, name ) -> !name.startsWith( "ignore" ); storeFiles = new StoreFiles( fs, pageCache, filter ); storeFiles.moveTo( src, tgt, logFiles ); assertFalse( fs.fileExists( a ) ); - assertFalse( pc.fileExists( b ) ); assertTrue( fs.fileExists( c ) ); - assertTrue( pc.fileExists( d ) ); assertTrue( fs.fileExists( new File( tgt, "a" ) ) ); - assertTrue( pc.fileExists( new File( tgt, "b" ) ) ); } @Test @@ -284,32 +238,23 @@ public void isEmptyMustFindFilesBothOnFileSystemAndPageCache() throws Exception File dir = getBaseDir(); File ignore = new File( dir, "ignore" ); File a = new File( dir, "a" ); - File b = new File( dir, "b" ); File c = new File( dir, "c" ); - File d = new File( dir, "d" ); createOnFileSystem( a ); createOnFileSystem( c ); createOnFileSystem( ignore ); - createOnPageCache( b ); - createOnPageCache( d ); - createOnPageCache( ignore ); FilenameFilter filter = ( directory, name ) -> !name.startsWith( "ignore" ); storeFiles = new StoreFiles( fs, pageCache, filter ); - List filesOnFilesystem = Arrays.asList( a, c ); - List fileOnFilesystem = Arrays.asList( a ); - List filesOnPageCache = Arrays.asList( b, d ); - List fileOnPageCache = Arrays.asList( b ); - List ingore = Arrays.asList( ignore ); + List filesOnFilesystem = asList( a, c ); + List fileOnFilesystem = singletonList( a ); + List ignoredList = singletonList( ignore ); assertFalse( storeFiles.isEmpty( dir, filesOnFilesystem ) ); assertFalse( storeFiles.isEmpty( dir, fileOnFilesystem ) ); - assertFalse( storeFiles.isEmpty( dir, filesOnPageCache ) ); - assertFalse( storeFiles.isEmpty( dir, fileOnPageCache ) ); assertTrue( storeFiles.isEmpty( dir, Collections.emptyList() ) ); - assertTrue( storeFiles.isEmpty( dir, ingore ) ); + assertTrue( storeFiles.isEmpty( dir, ignoredList ) ); } @Test @@ -323,7 +268,7 @@ public void mustReadStoreId() throws Exception long upgradeTime = rng.nextLong(); long upgradeTransactionId = rng.nextLong(); - createOnPageCache( neostore ); + createOnFileSystem( neostore ); MetaDataStore.setRecord( pageCache, neostore, Position.TIME, time ); MetaDataStore.setRecord( pageCache, neostore, Position.RANDOM_NUMBER, randomNumber ); diff --git a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/storecopy/StoreFilesWithRealFileSystemTest.java b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/storecopy/StoreFilesWithRealFileSystemTest.java index 9f9f9b75fa3f5..5fa218af31add 100644 --- a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/storecopy/StoreFilesWithRealFileSystemTest.java +++ b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/storecopy/StoreFilesWithRealFileSystemTest.java @@ -24,7 +24,6 @@ import org.neo4j.test.rule.PageCacheRule; import org.neo4j.test.rule.TestDirectory; import org.neo4j.test.rule.fs.DefaultFileSystemRule; -import org.neo4j.test.rule.fs.EphemeralFileSystemRule; public class StoreFilesWithRealFileSystemTest extends StoreFilesTest { @@ -34,11 +33,9 @@ protected void createRules() testDirectory = TestDirectory.testDirectory( StoreFilesWithRealFileSystemTest.class ); DefaultFileSystemRule defaultFileSystemRule = new DefaultFileSystemRule(); fileSystemRule = defaultFileSystemRule; - hiddenFileSystemRule = new EphemeralFileSystemRule(); pageCacheRule = new PageCacheRule( ); rules = RuleChain.outerRule( defaultFileSystemRule ) .around( testDirectory ) - .around( hiddenFileSystemRule ) .around( pageCacheRule ); } } diff --git a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/storecopy/StreamToDiskTest.java b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/storecopy/StreamToDiskTest.java index b3437a941ad29..d3b3b09a9f6ae 100644 --- a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/storecopy/StreamToDiskTest.java +++ b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/storecopy/StreamToDiskTest.java @@ -23,7 +23,8 @@ import org.junit.Test; import org.junit.rules.RuleChain; -import org.neo4j.io.pagecache.PageCache; +import java.io.File; + import org.neo4j.kernel.impl.index.labelscan.NativeLabelScanStore; import org.neo4j.kernel.impl.store.StoreType; import org.neo4j.kernel.monitoring.Monitors; @@ -31,11 +32,8 @@ import org.neo4j.test.rule.TestDirectory; import org.neo4j.test.rule.fs.EphemeralFileSystemRule; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyInt; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.verify; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import static org.neo4j.kernel.impl.storemigration.StoreFileType.STORE; public class StreamToDiskTest @@ -53,9 +51,8 @@ public class StreamToDiskTest public void shouldLetPageCacheHandleRecordStoresAndNativeLabelScanStoreFiles() throws Exception { // GIVEN - PageCache pageCache = spy( pageCacheRule.getPageCache( fs ) ); Monitors monitors = new Monitors(); - StreamToDiskProvider writerProvider = new StreamToDiskProvider( directory.absolutePath(), fs, pageCache, monitors ); + StreamToDiskProvider writerProvider = new StreamToDiskProvider( directory.absolutePath(), fs, monitors ); // WHEN for ( StoreType type : StoreType.values() ) @@ -63,18 +60,20 @@ public void shouldLetPageCacheHandleRecordStoresAndNativeLabelScanStoreFiles() t if ( type.isRecordStore() ) { String fileName = type.getStoreFile().fileName( STORE ); - writeAndVerifyWrittenThroughPageCache( pageCache, writerProvider, fileName ); + writeAndVerify( writerProvider, fileName ); } } - writeAndVerifyWrittenThroughPageCache( pageCache, writerProvider, NativeLabelScanStore.FILE_NAME ); + writeAndVerify( writerProvider, NativeLabelScanStore.FILE_NAME ); } - private void writeAndVerifyWrittenThroughPageCache( PageCache pageCache, StreamToDiskProvider writerProvider, String fileName ) throws Exception + private void writeAndVerify( StreamToDiskProvider writerProvider, String fileName ) throws Exception { try ( StoreFileStream acquire = writerProvider.acquire( fileName, 16 ) ) { acquire.write( DATA ); } - verify( pageCache ).map( eq( directory.file( fileName ) ), anyInt(), any() ); + File expectedFile = new File( directory.absolutePath(), fileName ); + assertTrue( "Streamed file created.", fs.fileExists( expectedFile ) ); + assertEquals( DATA.length, fs.getFileSize( expectedFile ) ); } } diff --git a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/storecopy/TestCatchupServer.java b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/storecopy/TestCatchupServer.java index 53fa9b5d9a6b9..42115f1a092de 100644 --- a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/storecopy/TestCatchupServer.java +++ b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/storecopy/TestCatchupServer.java @@ -20,12 +20,9 @@ package org.neo4j.causalclustering.catchup.storecopy; import java.util.function.BooleanSupplier; -import java.util.function.Function; import java.util.function.Supplier; import org.neo4j.causalclustering.catchup.CatchupProtocolServerInstaller; -import org.neo4j.causalclustering.catchup.CatchupServerHandler; -import org.neo4j.causalclustering.catchup.CatchupServerProtocol; import org.neo4j.causalclustering.catchup.RegularCatchupServerHandler; import org.neo4j.causalclustering.handlers.VoidPipelineWrapperFactory; import org.neo4j.causalclustering.identity.StoreId; @@ -95,7 +92,7 @@ private static ChildInitializer childInitializer( FileSystemAbstraction fileSyst RegularCatchupServerHandler catchupServerHandler = new RegularCatchupServerHandler( new Monitors(), logProvider, () -> storeId, dependencies.provideDependency( TransactionIdStore.class ), dependencies.provideDependency( LogicalTransactionStore.class ), - dataSource, availability, fileSystem, pageCache, storeCopyCheckPointMutex, null, checkPointer ); + dataSource, availability, fileSystem, storeCopyCheckPointMutex, null, checkPointer ); NettyPipelineBuilderFactory pipelineBuilder = new NettyPipelineBuilderFactory( VoidPipelineWrapperFactory.VOID_WRAPPER ); CatchupProtocolServerInstaller.Factory catchupProtocolServerInstaller = new CatchupProtocolServerInstaller.Factory( pipelineBuilder, logProvider, diff --git a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/readreplica/ReadReplicaStartupProcessTest.java b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/readreplica/ReadReplicaStartupProcessTest.java index 0e50647b962b3..77690bc9f3d6e 100644 --- a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/readreplica/ReadReplicaStartupProcessTest.java +++ b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/readreplica/ReadReplicaStartupProcessTest.java @@ -85,7 +85,6 @@ public void commonMocking() throws IOException FileSystemAbstraction fileSystemAbstraction = mock( FileSystemAbstraction.class ); when( fileSystemAbstraction.streamFilesRecursive( any( File.class ) ) ).thenAnswer( f -> Stream.empty() ); - when( pageCache.getCachedFileSystem() ).thenReturn( fileSystemAbstraction ); when( localDatabase.storeDir() ).thenReturn( storeDir ); when( localDatabase.storeId() ).thenReturn( localStoreId ); when( topologyService.allCoreServers() ).thenReturn( clusterTopology ); diff --git a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/scenarios/CoreToCoreCopySnapshotIT.java b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/scenarios/CoreToCoreCopySnapshotIT.java index 8c6938507a8a4..e6a074ccfb202 100644 --- a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/scenarios/CoreToCoreCopySnapshotIT.java +++ b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/scenarios/CoreToCoreCopySnapshotIT.java @@ -49,9 +49,6 @@ import static org.neo4j.causalclustering.scenarios.SampleData.createData; import static org.neo4j.helpers.collection.MapUtil.stringMap; -/** - * Note that this test is extended in the blockdevice repository. - */ public class CoreToCoreCopySnapshotIT { protected static final int NR_CORE_MEMBERS = 3; @@ -88,7 +85,6 @@ public void shouldBeAbleToDownloadLargerFreshSnapshot() throws Exception protected void deleteDirectoryRecursively( File directory, int id ) throws IOException { - // Extracted so the inheriting test in the block device repository can override it. id is used there. FileUtils.deleteRecursively( directory ); } diff --git a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/scenarios/ReadReplicaReplicationIT.java b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/scenarios/ReadReplicaReplicationIT.java index b3e150bc54d4e..0aa357679cc8f 100644 --- a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/scenarios/ReadReplicaReplicationIT.java +++ b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/scenarios/ReadReplicaReplicationIT.java @@ -59,12 +59,12 @@ import org.neo4j.graphdb.Transaction; import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.graphdb.security.WriteOperationsNotAllowedException; +import org.neo4j.internal.kernel.api.exceptions.TransactionFailureException; import org.neo4j.io.fs.DefaultFileSystemAbstraction; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; import org.neo4j.io.pagecache.monitoring.PageCacheCounters; import org.neo4j.kernel.AvailabilityGuard; -import org.neo4j.internal.kernel.api.exceptions.TransactionFailureException; import org.neo4j.kernel.api.labelscan.LabelScanStore; import org.neo4j.kernel.api.txtracking.TransactionIdTracker; import org.neo4j.kernel.impl.factory.GraphDatabaseFacade; @@ -105,12 +105,8 @@ import static org.neo4j.kernel.impl.store.MetaDataStore.Position.TIME; import static org.neo4j.test.assertion.Assert.assertEventually; -/** - * Note that this test is extended in the blockdevice repository. - */ public class ReadReplicaReplicationIT { - // This test is extended in the blockdevice repository, and these constants are required there as well. protected static final int NR_CORE_MEMBERS = 3; protected static final int NR_READ_REPLICAS = 1; diff --git a/enterprise/com/src/main/java/org/neo4j/com/storecopy/ExternallyManagedPageCache.java b/enterprise/com/src/main/java/org/neo4j/com/storecopy/ExternallyManagedPageCache.java index 40a6fe5664460..962cb3200b6e6 100644 --- a/enterprise/com/src/main/java/org/neo4j/com/storecopy/ExternallyManagedPageCache.java +++ b/enterprise/com/src/main/java/org/neo4j/com/storecopy/ExternallyManagedPageCache.java @@ -105,24 +105,12 @@ public long maxCachedPages() return delegate.maxCachedPages(); } - @Override - public FileSystemAbstraction getCachedFileSystem() - { - return delegate.getCachedFileSystem(); - } - @Override public void reportEvents() { delegate.reportEvents(); } - @Override - public boolean fileSystemSupportsFileOperations() - { - return delegate.fileSystemSupportsFileOperations(); - } - /** * Create a GraphDatabaseFactory that will build EmbeddedGraphDatabase instances that all use the given page cache. */ diff --git a/enterprise/com/src/main/java/org/neo4j/com/storecopy/FileMoveAction.java b/enterprise/com/src/main/java/org/neo4j/com/storecopy/FileMoveAction.java index 1787bb3d0883d..bcf1fc63786c8 100644 --- a/enterprise/com/src/main/java/org/neo4j/com/storecopy/FileMoveAction.java +++ b/enterprise/com/src/main/java/org/neo4j/com/storecopy/FileMoveAction.java @@ -24,10 +24,6 @@ import java.nio.file.CopyOption; import java.nio.file.Files; import java.nio.file.Path; -import java.util.Optional; - -import org.neo4j.io.fs.FileHandle; -import org.neo4j.io.pagecache.PageCache; public interface FileMoveAction { @@ -35,29 +31,6 @@ public interface FileMoveAction File file(); - static FileMoveAction copyViaPageCache( File file, PageCache pageCache ) - { - return new FileMoveAction() - { - @Override - public void move( File toDir, CopyOption... copyOptions ) throws IOException - { - Optional handle = pageCache.getCachedFileSystem().streamFilesRecursive( file ).findAny(); - boolean directoryExistsInCachedSystem = handle.isPresent(); - if ( directoryExistsInCachedSystem ) - { - handle.get().rename( new File( toDir, file.getName() ), copyOptions ); - } - } - - @Override - public File file() - { - return file; - } - }; - } - static FileMoveAction copyViaFileSystem( File file, File basePath ) { Path base = basePath.toPath(); diff --git a/enterprise/com/src/main/java/org/neo4j/com/storecopy/FileMoveProvider.java b/enterprise/com/src/main/java/org/neo4j/com/storecopy/FileMoveProvider.java index 3bd2486fd3592..62dac209fdc08 100644 --- a/enterprise/com/src/main/java/org/neo4j/com/storecopy/FileMoveProvider.java +++ b/enterprise/com/src/main/java/org/neo4j/com/storecopy/FileMoveProvider.java @@ -26,29 +26,15 @@ import java.util.stream.Stream; import org.neo4j.io.fs.FileSystemAbstraction; -import org.neo4j.io.pagecache.PageCache; -import org.neo4j.kernel.impl.store.StoreType; -import static java.util.Optional.ofNullable; import static java.util.stream.Collectors.toList; -import static java.util.stream.Stream.empty; public class FileMoveProvider { - private final FileMoveActionInformer fileMoveActionInformer; - private final PageCache pageCache; private final FileSystemAbstraction fs; - public FileMoveProvider( PageCache pageCache, FileSystemAbstraction fs ) + public FileMoveProvider( FileSystemAbstraction fs ) { - this( pageCache, StoreType::canBeManagedByPageCache, fs ); - } - - public FileMoveProvider( PageCache pageCache, FileMoveActionInformer fileMoveActionInformer, - FileSystemAbstraction fs ) - { - this.pageCache = pageCache; - this.fileMoveActionInformer = fileMoveActionInformer; this.fs = fs; } @@ -130,37 +116,24 @@ private Stream expandTraverseFiles( File dir, File basePath ) private boolean isFile( File file ) { - if ( fileMoveActionInformer.shouldBeManagedByPageCache( file.getName() ) ) - { - return !pageCache.getCachedFileSystem().isDirectory( file ); - } return !fs.isDirectory( file ); } private boolean isDirectory( File file ) { - if ( fileMoveActionInformer.shouldBeManagedByPageCache( file.getName() ) ) - { - return pageCache.getCachedFileSystem().isDirectory( file ); - } return fs.isDirectory( file ); } private List listFiles( File dir ) { - File[] cachedFiles = pageCache.getCachedFileSystem().listFiles( dir ); File[] fsaFiles = fs.listFiles( dir ); - if ( cachedFiles == null && fsaFiles == null ) + if ( fsaFiles == null ) { // This probably means 'dir' is actually a file, or it does not exist. return null; } - Stream files = Stream.concat( - ofNullable( cachedFiles ).map( Arrays::stream ).orElse( empty() ), - ofNullable( fsaFiles ).map( Arrays::stream ).orElse( empty() ) ); - - return files.distinct().collect( toList() ); + return Arrays.stream( fsaFiles ).distinct().collect( toList() ); } /** @@ -170,10 +143,6 @@ private List listFiles( File dir ) */ private FileMoveAction copyFileCorrectly( File fileToMove, File basePath ) { - if ( fileMoveActionInformer.shouldBeManagedByPageCache( fileToMove.getName() ) ) - { - return FileMoveAction.copyViaPageCache( fileToMove, pageCache ); - } return FileMoveAction.copyViaFileSystem( fileToMove, basePath ); } } diff --git a/enterprise/com/src/main/java/org/neo4j/com/storecopy/StoreCopyClient.java b/enterprise/com/src/main/java/org/neo4j/com/storecopy/StoreCopyClient.java index 2a55093fe8970..91881d2c53f8a 100644 --- a/enterprise/com/src/main/java/org/neo4j/com/storecopy/StoreCopyClient.java +++ b/enterprise/com/src/main/java/org/neo4j/com/storecopy/StoreCopyClient.java @@ -23,8 +23,6 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.nio.channels.ReadableByteChannel; -import java.util.ArrayList; -import java.util.List; import java.util.concurrent.atomic.AtomicLong; import java.util.function.Function; import java.util.stream.Stream; @@ -94,8 +92,7 @@ public interface StoreCopyRequester public StoreCopyClient( File storeDir, Config config, Iterable> kernelExtensions, LogProvider logProvider, FileSystemAbstraction fs, PageCache pageCache, StoreCopyClientMonitor monitor, boolean forensics ) { - this( storeDir, config, kernelExtensions, logProvider, fs, pageCache, monitor, forensics, new FileMoveProvider( pageCache, - fs ) ); + this( storeDir, config, kernelExtensions, logProvider, fs, pageCache, monitor, forensics, new FileMoveProvider( fs ) ); } public StoreCopyClient( File storeDir, Config config, Iterable> kernelExtensions, LogProvider logProvider, @@ -118,16 +115,11 @@ public void copyStore( StoreCopyRequester requester, CancellationRequest cancell File tempStore = new File( storeDir, StoreUtil.TEMP_COPY_DIRECTORY_NAME ); try { - // The ToFileStoreWriter will add FileMoveActions for *RecordStores* that have to be - // *moved via the PageCache*! - // We have to move these files via the page cache, because that is the *only way* that we can communicate - // with any block storage that might have been configured for this instance. - List moveActions = new ArrayList<>(); cleanDirectory( tempStore ); // Request store files and transactions that will need recovery monitor.startReceivingStoreFiles(); - ToFileStoreWriter storeWriter = new ToFileStoreWriter( tempStore, fs, monitor, pageCache, moveActions ); + ToFileStoreWriter storeWriter = new ToFileStoreWriter( tempStore, fs, monitor ); try ( Response response = requester.copyStore( decorateWithProgressIndicator( storeWriter ) ) ) { monitor.finishReceivingStoreFiles(); @@ -147,10 +139,10 @@ public void copyStore( StoreCopyRequester requester, CancellationRequest cancell recoverDatabase( tempStore ); // All is well, move the streamed files to the real store directory. - // Start with the files written through the page cache. Should only be record store files. + // Should only be record store files. // Note that the stream is lazy, so the file system traversal won't happen until *after* the store files // have been moved. Thus we ensure that we only attempt to move them once. - moveFromTemporaryLocationToCorrect( moveActions, tempStore, moveAfterCopy ); + moveFromTemporaryLocationToCorrect( tempStore, moveAfterCopy ); } finally { @@ -159,13 +151,11 @@ public void copyStore( StoreCopyRequester requester, CancellationRequest cancell } } - private void moveFromTemporaryLocationToCorrect( - List storeFileMoveActions, File tempStore, MoveAfterCopy moveAfterCopy ) throws Exception + private void moveFromTemporaryLocationToCorrect( File tempStore, MoveAfterCopy moveAfterCopy ) throws Exception { LogFiles logFiles = LogFilesBuilder.activeFilesBuilder( storeDir, fs, pageCache ).withConfig( config ).build(); - Stream moveActionStream = - Stream.concat( storeFileMoveActions.stream(), fileMoveProvider.traverseForMoving( tempStore ) ); + Stream moveActionStream = fileMoveProvider.traverseForMoving( tempStore ) ; Function destinationMapper = file -> logFiles.isLogFile( file ) ? logFiles.logFilesDirectory() : storeDir; moveAfterCopy.move( moveActionStream, tempStore, destinationMapper ); diff --git a/enterprise/com/src/main/java/org/neo4j/com/storecopy/StoreCopyServer.java b/enterprise/com/src/main/java/org/neo4j/com/storecopy/StoreCopyServer.java index 92d6ae4f3e6db..502668415fe39 100644 --- a/enterprise/com/src/main/java/org/neo4j/com/storecopy/StoreCopyServer.java +++ b/enterprise/com/src/main/java/org/neo4j/com/storecopy/StoreCopyServer.java @@ -23,7 +23,6 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.nio.channels.ReadableByteChannel; -import java.util.Optional; import org.neo4j.com.RequestContext; import org.neo4j.com.Response; @@ -34,8 +33,6 @@ import org.neo4j.io.ByteUnit; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.fs.OpenMode; -import org.neo4j.io.pagecache.PageCache; -import org.neo4j.io.pagecache.PagedFile; import org.neo4j.kernel.NeoStoreDataSource; import org.neo4j.kernel.impl.transaction.log.checkpoint.CheckPointer; import org.neo4j.kernel.impl.transaction.log.checkpoint.SimpleTriggerInfo; @@ -121,11 +118,10 @@ public void finishStreamingTransactions( long endTxId, String storeCopyIdentifie private final FileSystemAbstraction fileSystem; private final File storeDirectory; private final Monitor monitor; - private final PageCache pageCache; private final StoreCopyCheckPointMutex mutex; public StoreCopyServer( NeoStoreDataSource dataSource, CheckPointer checkPointer, FileSystemAbstraction fileSystem, - File storeDirectory, Monitor monitor, PageCache pageCache, StoreCopyCheckPointMutex mutex ) + File storeDirectory, Monitor monitor, StoreCopyCheckPointMutex mutex ) { this.dataSource = dataSource; this.checkPointer = checkPointer; @@ -133,7 +129,6 @@ public StoreCopyServer( NeoStoreDataSource dataSource, CheckPointer checkPointer this.mutex = mutex; this.storeDirectory = getMostCanonicalFile( storeDirectory ); this.monitor = monitor; - this.pageCache = pageCache; } public Monitor monitor() @@ -176,27 +171,6 @@ public RequestContext flushStoresAndStreamStoreFiles( String triggerName, StoreW boolean isLogFile = meta.isLogFile(); int recordSize = meta.recordSize(); - if ( !pageCache.fileSystemSupportsFileOperations() ) - { - // Read from paged file if mapping exists. Otherwise read through file system. - // A file is mapped if it is a store, and we have a running database, which will be the case for - // both online backup, and when we are the master of an HA cluster. - final Optional optionalPagedFile = pageCache.getExistingMapping( file ); - if ( optionalPagedFile.isPresent() ) - { - try ( PagedFile pagedFile = optionalPagedFile.get() ) - { - long fileSize = pagedFile.fileSize(); - try ( ReadableByteChannel fileChannel = pagedFile.openReadableByteChannel() ) - { - doWrite( writer, temporaryBuffer, file, recordSize, fileChannel, fileSize, - storeCopyIdentifier, false ); - } - continue; - } - } - } - try ( ReadableByteChannel fileChannel = fileSystem.open( file, OpenMode.READ ) ) { long fileSize = fileSystem.getFileSize( file ); diff --git a/enterprise/com/src/main/java/org/neo4j/com/storecopy/StoreUtil.java b/enterprise/com/src/main/java/org/neo4j/com/storecopy/StoreUtil.java index 6a2b624a8ec30..6f5a1e6489014 100644 --- a/enterprise/com/src/main/java/org/neo4j/com/storecopy/StoreUtil.java +++ b/enterprise/com/src/main/java/org/neo4j/com/storecopy/StoreUtil.java @@ -24,14 +24,8 @@ import java.io.File; import java.io.FileFilter; import java.io.IOException; -import java.util.function.Consumer; -import java.util.stream.Stream; import org.neo4j.io.fs.FileUtils; -import org.neo4j.io.fs.FileHandle; -import org.neo4j.io.pagecache.PageCache; - -import static org.neo4j.io.fs.FileHandle.HANDLE_DELETE; public class StoreUtil { @@ -51,31 +45,17 @@ public class StoreUtil } return !isBranchedDataRootDirectory( file ) && !isTemporaryCopy( file ); }; - private static final FileFilter DEEP_STORE_FILE_FILTER = file -> - { - for ( String directory : DONT_MOVE_DIRECTORIES ) - { - if ( file.getPath().contains( directory ) ) - { - return false; - } - } - return !isPartOfBranchedDataRootDirectory( file ); - }; private StoreUtil() { } - public static void cleanStoreDir( File storeDir, PageCache pageCache ) throws IOException + public static void cleanStoreDir( File storeDir ) throws IOException { for ( File file : relevantDbFiles( storeDir ) ) { FileUtils.deleteRecursively( file ); } - - pageCache.getCachedFileSystem().streamFilesRecursive( storeDir ) - .filter( fh -> DEEP_STORE_FILE_FILTER.accept( fh.getFile() ) ).forEach( HANDLE_DELETE ); } public static File newBranchedDataDir( File storeDir ) @@ -85,36 +65,17 @@ public static File newBranchedDataDir( File storeDir ) return result; } - public static void moveAwayDb( File storeDir, File branchedDataDir, PageCache pageCache ) throws IOException + public static void moveAwayDb( File storeDir, File branchedDataDir ) throws IOException { for ( File file : relevantDbFiles( storeDir ) ) { FileUtils.moveFileToDirectory( file, branchedDataDir ); } - - moveAwayDbWithPageCache( storeDir, branchedDataDir, pageCache, DEEP_STORE_FILE_FILTER ); - } - - public static void moveAwayDbWithPageCache( File from, File to, PageCache pageCache, FileFilter filter ) - { - final Stream fileHandleStream; - try - { - fileHandleStream = pageCache.getCachedFileSystem().streamFilesRecursive( from ); - } - catch ( IOException e ) - { - // Directory does not exist, has possibly been moved with file system previous to this call. - return; - } - final Consumer handleRename = FileHandle.handleRenameBetweenDirectories( from, to ); - fileHandleStream.filter( fh -> filter.accept( fh.getFile() ) ).forEach( handleRename ); } - public static void deleteRecursive( File storeDir, PageCache pageCache ) throws IOException + public static void deleteRecursive( File storeDir ) throws IOException { FileUtils.deleteRecursively( storeDir ); - pageCache.getCachedFileSystem().streamFilesRecursive( storeDir ).forEach( HANDLE_DELETE ); } public static boolean isBranchedDataDirectory( File file ) @@ -153,8 +114,4 @@ private static boolean isTemporaryCopy( File file ) return file.isDirectory() && file.getName().equals( TEMP_COPY_DIRECTORY_NAME ); } - private static boolean isPartOfBranchedDataRootDirectory( File file ) - { - return file.getPath().contains( BRANCH_SUBDIRECTORY ); - } } diff --git a/enterprise/com/src/main/java/org/neo4j/com/storecopy/ToFileStoreWriter.java b/enterprise/com/src/main/java/org/neo4j/com/storecopy/ToFileStoreWriter.java index a212346444d23..7da119db6c62a 100644 --- a/enterprise/com/src/main/java/org/neo4j/com/storecopy/ToFileStoreWriter.java +++ b/enterprise/com/src/main/java/org/neo4j/com/storecopy/ToFileStoreWriter.java @@ -24,34 +24,22 @@ import java.nio.ByteBuffer; import java.nio.channels.ReadableByteChannel; import java.nio.channels.WritableByteChannel; -import java.util.List; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.fs.StoreChannel; -import org.neo4j.io.pagecache.PageCache; -import org.neo4j.io.pagecache.PagedFile; -import org.neo4j.kernel.impl.store.StoreType; - -import static java.nio.file.StandardOpenOption.CREATE; -import static java.nio.file.StandardOpenOption.WRITE; -import static org.neo4j.kernel.impl.store.format.RecordFormat.NO_RECORD_SIZE; public class ToFileStoreWriter implements StoreWriter { private final File basePath; private final FileSystemAbstraction fs; private final StoreCopyClientMonitor monitor; - private final PageCache pageCache; - private final List fileMoveActions; public ToFileStoreWriter( File graphDbStoreDir, FileSystemAbstraction fs, - StoreCopyClientMonitor storeCopyClientMonitor, PageCache pageCache, List fileMoveActions ) + StoreCopyClientMonitor storeCopyClientMonitor ) { this.basePath = graphDbStoreDir; this.fs = fs; this.monitor = storeCopyClientMonitor; - this.pageCache = pageCache; - this.fileMoveActions = fileMoveActions; } @Override @@ -69,18 +57,6 @@ public long write( String path, ReadableByteChannel data, ByteBuffer temporaryBu monitor.startReceivingStoreFile( fullFilePath ); try { - // Note that we don't bother checking if the page cache already has a mapping for the given file. - // The reason is that we are copying to a temporary store location, and then we'll move the files later. - if ( !pageCache.fileSystemSupportsFileOperations() && StoreType.canBeManagedByPageCache( file.getName() ) ) - { - int filePageSize = filePageSize( requiredElementAlignment ); - try ( PagedFile pagedFile = pageCache.map( file, filePageSize, CREATE, WRITE ) ) - { - final long written = writeDataThroughPageCache( pagedFile, data, temporaryBuffer, hasData ); - addPageCacheMoveAction( file ); - return written; - } - } // We don't add file move actions for these files. The reason is that we will perform the file moves // *after* we have done recovery on the store, and this may delete some files, and add other files. return writeDataThroughFileSystem( file, data, temporaryBuffer, hasData ); @@ -96,22 +72,6 @@ public long write( String path, ReadableByteChannel data, ByteBuffer temporaryBu } } - // As only the page cache know towards which device (block device or normal file system) it is working, we use - // the page cache later on when we want to move the files written through the page cache. - private void addPageCacheMoveAction( File file ) - { - fileMoveActions.add( FileMoveAction.copyViaPageCache( file, pageCache ) ); - } - - private int filePageSize( int alignment ) - { - // We know we are dealing with a record store at this point, so the required alignment is the record size, - // and we can use this to do the page size calculation in the same way as the stores would. - final int pageCacheSize = pageCache.pageSize(); - return (alignment == NO_RECORD_SIZE) ? pageCacheSize - : (pageCacheSize - (pageCacheSize % alignment)); - } - private long writeDataThroughFileSystem( File file, ReadableByteChannel data, ByteBuffer temporaryBuffer, boolean hasData ) throws IOException { @@ -121,15 +81,6 @@ private long writeDataThroughFileSystem( File file, ReadableByteChannel data, By } } - private long writeDataThroughPageCache( PagedFile pagedFile, ReadableByteChannel data, ByteBuffer temporaryBuffer, - boolean hasData ) throws IOException - { - try ( WritableByteChannel channel = pagedFile.openWritableByteChannel() ) - { - return writeData( data, temporaryBuffer, hasData, channel ); - } - } - private long writeData( ReadableByteChannel data, ByteBuffer temporaryBuffer, boolean hasData, WritableByteChannel channel ) throws IOException { diff --git a/enterprise/com/src/test/java/org/neo4j/com/storecopy/FileMoveProviderTest.java b/enterprise/com/src/test/java/org/neo4j/com/storecopy/FileMoveProviderTest.java index c789433ed9a37..4ae14ecb9ac60 100644 --- a/enterprise/com/src/test/java/org/neo4j/com/storecopy/FileMoveProviderTest.java +++ b/enterprise/com/src/test/java/org/neo4j/com/storecopy/FileMoveProviderTest.java @@ -29,46 +29,28 @@ import java.io.FileReader; import java.io.FileWriter; import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.List; import java.util.function.Consumer; -import java.util.stream.Collectors; -import org.neo4j.graphdb.mockfs.EphemeralFileSystemAbstraction; import org.neo4j.io.fs.DefaultFileSystemAbstraction; -import org.neo4j.io.fs.StoreChannel; -import org.neo4j.io.pagecache.PageCache; -import org.neo4j.test.rule.PageCacheRule; import org.neo4j.test.rule.TestDirectory; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; public class FileMoveProviderTest { private DefaultFileSystemAbstraction defaultFileSystemAbstraction = new DefaultFileSystemAbstraction(); - private EphemeralFileSystemAbstraction ephemeralFileSystemAbstraction = new EphemeralFileSystemAbstraction(); @Rule public TestDirectory testDirectory = TestDirectory.testDirectory( defaultFileSystemAbstraction ); private FileMoveProvider subject; - public PageCacheRule pageCacheRule = new PageCacheRule(); - private PageCache pageCache; - private FileMoveActionInformer fileMoveActionInformer; - @Before public void setup() { - pageCache = pageCacheRule.getPageCache( ephemeralFileSystemAbstraction ); - fileMoveActionInformer = mock( FileMoveActionInformer.class ); - subject = new FileMoveProvider( pageCache, fileMoveActionInformer, defaultFileSystemAbstraction ); + subject = new FileMoveProvider( defaultFileSystemAbstraction ); } @Test @@ -150,42 +132,6 @@ public void moveNestedFiles() throws IOException assertEquals( "This is the file contained in directory B", readFromFile( targetFileTwo ) ); } - @Test - public void filesAreMovedViaPageCacheWhenNecessary() throws IOException - { - // given there is a file on the default file system - File parentDirectory = testDirectory.cleanDirectory( "parent" ); - File aNormalFile = new File( parentDirectory, "aNormalFile.A" ); - assertTrue( aNormalFile.createNewFile() ); - - // and we have an expected target directory - File targetDirectory = testDirectory.cleanDirectory( "targetDirectory" ); - pageCache.getCachedFileSystem().mkdirs( targetDirectory ); - - // and there is also a file on the block device - File aPageCacheFile = new File( parentDirectory, "aBlockCopyFile.B" ); - pageCache.getCachedFileSystem().mkdirs( parentDirectory ); - StoreChannel storeChannel = pageCache.getCachedFileSystem().create( aPageCacheFile ); - storeChannel.write( ByteBuffer.allocate( 20 ).putChar( 'a' ).putChar( 'b' ) ); - - // and some of these files are handled by the page cache - when( fileMoveActionInformer.shouldBeManagedByPageCache( any() ) ).thenReturn( false ); - when( fileMoveActionInformer.shouldBeManagedByPageCache( eq( aPageCacheFile.getName() ) ) ).thenReturn( true ); - - // when the files are copied to target location - List moveActions = - subject.traverseForMoving( parentDirectory ).collect( Collectors.toList() );//.forEach( moveToDirectory( targetDirectory ) ); - moveActions.forEach( moveToDirectory( targetDirectory ) ); - - // then some files are copied over the default file system - File expectedNormalCopy = new File( targetDirectory, aNormalFile.getName() ); - assertTrue( expectedNormalCopy.exists() ); - - // and correct files are copied over the page cache - File expectedPageCacheCopy = new File( targetDirectory, aPageCacheFile.getName() ); - assertTrue( expectedPageCacheCopy.toString(), pageCache.getCachedFileSystem().fileExists( expectedPageCacheCopy ) ); - } - @Test public void filesAreMovedBeforeDirectories() throws IOException { @@ -240,7 +186,6 @@ private void writeToFile( File output, String input ) throws IOException try ( BufferedWriter bw = new BufferedWriter( new FileWriter( output ) ) ) { bw.write( input ); - bw.close(); } } } diff --git a/enterprise/com/src/test/java/org/neo4j/com/storecopy/StoreCopyClientTest.java b/enterprise/com/src/test/java/org/neo4j/com/storecopy/StoreCopyClientTest.java index 870e34dedfd61..c70bf89dc2cb7 100644 --- a/enterprise/com/src/test/java/org/neo4j/com/storecopy/StoreCopyClientTest.java +++ b/enterprise/com/src/test/java/org/neo4j/com/storecopy/StoreCopyClientTest.java @@ -42,7 +42,6 @@ import org.neo4j.helpers.Service; import org.neo4j.helpers.collection.Iterators; import org.neo4j.io.fs.FileSystemAbstraction; -import org.neo4j.io.pagecache.DelegatingPageCache; import org.neo4j.io.pagecache.PageCache; import org.neo4j.kernel.NeoStoreDataSource; import org.neo4j.kernel.configuration.Config; @@ -96,7 +95,7 @@ public class StoreCopyClientTest @Parameters public static StoreCopyRequestFactory[] data() { - return new StoreCopyRequestFactory[]{LocalStoreCopyRequester::new, LocalStoreCopyRequesterForcePageCache::new}; + return new StoreCopyRequestFactory[]{LocalStoreCopyRequester::new}; } @Parameter @@ -471,10 +470,8 @@ public Response copyStore( StoreWriter writer ) CheckPointer checkPointer = original.getDependencyResolver().resolveDependency( CheckPointer.class ); - PageCache pageCache = getPageCache(); - RequestContext requestContext = new StoreCopyServer( neoStoreDataSource, checkPointer, fs, - originalDir, new Monitors().newMonitor( StoreCopyServer.Monitor.class ), pageCache, + originalDir, new Monitors().newMonitor( StoreCopyServer.Monitor.class ), new StoreCopyCheckPointMutex() ) .flushStoresAndStreamStoreFiles( "test", writer, includeLogs ); @@ -497,26 +494,4 @@ public void done() verify( response, times( 1 ) ).close(); } } - - private static class LocalStoreCopyRequesterForcePageCache extends LocalStoreCopyRequester - { - - LocalStoreCopyRequesterForcePageCache( GraphDatabaseAPI original, File originalDir, FileSystemAbstraction fs, boolean includeLogs ) - { - super( original, originalDir, fs, includeLogs ); - } - - @Override - protected PageCache getPageCache() - { - return new DelegatingPageCache( super.getPageCache() ) - { - @Override - public boolean fileSystemSupportsFileOperations() - { - return false; - } - }; - } - } } diff --git a/enterprise/com/src/test/java/org/neo4j/com/storecopy/ToFileStoreWriterTest.java b/enterprise/com/src/test/java/org/neo4j/com/storecopy/ToFileStoreWriterTest.java index 65d14d8be8010..e1247ce5a93da 100644 --- a/enterprise/com/src/test/java/org/neo4j/com/storecopy/ToFileStoreWriterTest.java +++ b/enterprise/com/src/test/java/org/neo4j/com/storecopy/ToFileStoreWriterTest.java @@ -26,13 +26,9 @@ import java.io.File; import java.io.IOException; import java.nio.ByteBuffer; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; import org.neo4j.com.DataProducer; -import org.neo4j.io.pagecache.PageCache; import org.neo4j.kernel.impl.index.labelscan.NativeLabelScanStore; import org.neo4j.kernel.impl.store.StoreType; import org.neo4j.test.rule.PageCacheRule; @@ -41,11 +37,6 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyInt; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.verify; import static org.neo4j.kernel.impl.storemigration.StoreFileType.STORE; public class ToFileStoreWriterTest @@ -61,10 +52,8 @@ public class ToFileStoreWriterTest public void shouldLetPageCacheHandleRecordStoresAndNativeLabelScanStoreFiles() throws Exception { // GIVEN - List actions = new ArrayList<>(); - PageCache pageCache = spy( pageCacheRule.getPageCache( fs ) ); ToFileStoreWriter writer = new ToFileStoreWriter( directory.absolutePath(), fs, - new StoreCopyClientMonitor.Adapter(), pageCache, actions ); + new StoreCopyClientMonitor.Adapter() ); ByteBuffer tempBuffer = ByteBuffer.allocate( 128 ); // WHEN @@ -73,10 +62,10 @@ public void shouldLetPageCacheHandleRecordStoresAndNativeLabelScanStoreFiles() t if ( type.isRecordStore() ) { String fileName = type.getStoreFile().fileName( STORE ); - writeAndVerifyWrittenThroughPageCache( pageCache, writer, tempBuffer, fileName ); + writeAndVerify( writer, tempBuffer, fileName ); } } - writeAndVerifyWrittenThroughPageCache( pageCache, writer, tempBuffer, NativeLabelScanStore.FILE_NAME ); + writeAndVerify( writer, tempBuffer, NativeLabelScanStore.FILE_NAME ); } @Test @@ -95,10 +84,8 @@ public void startReceivingStoreFile( String file ) }; // and - List actions = new ArrayList<>(); - PageCache pageCache = spy( pageCacheRule.getPageCache( fs ) ); ToFileStoreWriter writer = new ToFileStoreWriter( directory.absolutePath(), fs, - monitor, pageCache, actions ); + monitor ); ByteBuffer tempBuffer = ByteBuffer.allocate( 128 ); // when @@ -108,12 +95,12 @@ public void startReceivingStoreFile( String file ) assertTrue( wasActivated.get() ); } - private void writeAndVerifyWrittenThroughPageCache( PageCache pageCache, ToFileStoreWriter writer, - ByteBuffer tempBuffer, String fileName ) + private void writeAndVerify( ToFileStoreWriter writer, ByteBuffer tempBuffer, String fileName ) throws IOException { - File expected = new File( directory.absolutePath(), fileName ); + File expectedFile = new File( directory.absolutePath(), fileName ); writer.write( fileName, new DataProducer( 16 ), tempBuffer, true, 16 ); - verify( pageCache ).map( eq( expected ), anyInt(), any() ); + assertTrue( "File created by writer should exist." , fs.fileExists( expectedFile ) ); + assertEquals( 16, fs.getFileSize( expectedFile ) ); } } diff --git a/enterprise/ha/src/main/java/org/neo4j/kernel/ha/BranchedDataMigrator.java b/enterprise/ha/src/main/java/org/neo4j/kernel/ha/BranchedDataMigrator.java index 4b9df1584c363..f0e324e1bfce3 100644 --- a/enterprise/ha/src/main/java/org/neo4j/kernel/ha/BranchedDataMigrator.java +++ b/enterprise/ha/src/main/java/org/neo4j/kernel/ha/BranchedDataMigrator.java @@ -22,9 +22,9 @@ import java.io.File; import java.io.IOException; +import org.neo4j.com.storecopy.StoreUtil; import org.neo4j.io.fs.FileUtils; import org.neo4j.io.pagecache.PageCache; -import org.neo4j.com.storecopy.StoreUtil; import org.neo4j.kernel.lifecycle.LifecycleAdapter; public class BranchedDataMigrator extends LifecycleAdapter @@ -70,7 +70,6 @@ private void migrateBranchedDataDirectoriesToRootDirectory() try { FileUtils.moveFile( oldBranchedDir, targetDir ); - StoreUtil.moveAwayDbWithPageCache( oldBranchedDir, targetDir, pageCache, f -> true ); } catch ( IOException e ) { diff --git a/enterprise/ha/src/main/java/org/neo4j/kernel/ha/BranchedDataPolicy.java b/enterprise/ha/src/main/java/org/neo4j/kernel/ha/BranchedDataPolicy.java index 841146f905fe2..b730c38e069df 100644 --- a/enterprise/ha/src/main/java/org/neo4j/kernel/ha/BranchedDataPolicy.java +++ b/enterprise/ha/src/main/java/org/neo4j/kernel/ha/BranchedDataPolicy.java @@ -43,7 +43,7 @@ public void handle( File storeDir, PageCache pageCache, LogService logService ) Log msgLog = logService.getInternalLog( getClass() ); File branchedDataDir = newBranchedDataDir( storeDir ); msgLog.debug( "Moving store from " + storeDir + " to " + branchedDataDir ); - moveAwayDb( storeDir, branchedDataDir, pageCache ); + moveAwayDb( storeDir, branchedDataDir ); } }, keep_last @@ -55,12 +55,12 @@ public void handle( File storeDir, PageCache pageCache, LogService logService ) File branchedDataDir = newBranchedDataDir( storeDir ); msgLog.debug( "Moving store from " + storeDir + " to " + branchedDataDir ); - moveAwayDb( storeDir, branchedDataDir, pageCache ); + moveAwayDb( storeDir, branchedDataDir ); for ( File file : getBranchedDataRootDirectory( storeDir ).listFiles() ) { if ( isBranchedDataDirectory( file ) && !file.equals( branchedDataDir ) ) { - deleteRecursive( file, pageCache ); + deleteRecursive( file ); } } } @@ -72,7 +72,7 @@ public void handle( File storeDir, PageCache pageCache, LogService logService ) { Log msgLog = logService.getInternalLog( getClass() ); msgLog.debug( "Removing store " + storeDir ); - cleanStoreDir( storeDir, pageCache ); + cleanStoreDir( storeDir ); } }; diff --git a/enterprise/ha/src/main/java/org/neo4j/kernel/ha/cluster/DefaultMasterImplSPI.java b/enterprise/ha/src/main/java/org/neo4j/kernel/ha/cluster/DefaultMasterImplSPI.java index 3ddb26e4509e6..8a948808cc3d3 100644 --- a/enterprise/ha/src/main/java/org/neo4j/kernel/ha/cluster/DefaultMasterImplSPI.java +++ b/enterprise/ha/src/main/java/org/neo4j/kernel/ha/cluster/DefaultMasterImplSPI.java @@ -27,10 +27,10 @@ import org.neo4j.com.storecopy.ResponsePacker; import org.neo4j.com.storecopy.StoreCopyServer; import org.neo4j.com.storecopy.StoreWriter; +import org.neo4j.internal.kernel.api.exceptions.TransactionFailureException; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; import org.neo4j.kernel.NeoStoreDataSource; -import org.neo4j.internal.kernel.api.exceptions.TransactionFailureException; import org.neo4j.kernel.ha.TransactionChecksumLookup; import org.neo4j.kernel.ha.com.master.MasterImpl; import org.neo4j.kernel.ha.id.IdAllocation; @@ -168,8 +168,7 @@ public long getTransactionChecksum( long txId ) throws IOException public RequestContext flushStoresAndStreamStoreFiles( StoreWriter writer ) { StoreCopyServer streamer = new StoreCopyServer( neoStoreDataSource, checkPointer, fileSystem, storeDir, - monitors.newMonitor( StoreCopyServer.Monitor.class, StoreCopyServer.class ), - pageCache, mutex ); + monitors.newMonitor( StoreCopyServer.Monitor.class, StoreCopyServer.class ), mutex ); return streamer.flushStoresAndStreamStoreFiles( STORE_COPY_CHECKPOINT_TRIGGER, writer, false ); } diff --git a/enterprise/ha/src/main/java/org/neo4j/kernel/ha/cluster/SwitchToSlave.java b/enterprise/ha/src/main/java/org/neo4j/kernel/ha/cluster/SwitchToSlave.java index fe290d8055f80..ec166cd86e71f 100644 --- a/enterprise/ha/src/main/java/org/neo4j/kernel/ha/cluster/SwitchToSlave.java +++ b/enterprise/ha/src/main/java/org/neo4j/kernel/ha/cluster/SwitchToSlave.java @@ -491,7 +491,7 @@ abstract void checkDataConsistency( MasterClient masterClient, TransactionIdStor void cleanStoreDir() throws IOException { // Tests verify that this method is called - StoreUtil.cleanStoreDir( storeDir, pageCache ); + StoreUtil.cleanStoreDir( storeDir ); } void stopServices() throws Exception diff --git a/enterprise/ha/src/test/java/org/neo4j/kernel/ha/cluster/SwitchToSlaveBranchThenCopyTest.java b/enterprise/ha/src/test/java/org/neo4j/kernel/ha/cluster/SwitchToSlaveBranchThenCopyTest.java index ddfaa0a0794cf..ef92702e3f7c8 100644 --- a/enterprise/ha/src/test/java/org/neo4j/kernel/ha/cluster/SwitchToSlaveBranchThenCopyTest.java +++ b/enterprise/ha/src/test/java/org/neo4j/kernel/ha/cluster/SwitchToSlaveBranchThenCopyTest.java @@ -279,7 +279,6 @@ private SwitchToSlaveBranchThenCopy newSwitchToSlaveSpy() throws Exception FileSystemAbstraction fileSystemAbstraction = mock( FileSystemAbstraction.class ); when( fileSystemAbstraction.streamFilesRecursive( any( File.class ) ) ) .thenAnswer( f -> Stream.empty() ); - when( pageCacheMock.getCachedFileSystem() ).thenReturn( fileSystemAbstraction ); StoreCopyClient storeCopyClient = mock( StoreCopyClient.class ); diff --git a/enterprise/ha/src/test/java/org/neo4j/kernel/ha/cluster/SwitchToSlaveCopyThenBranchTest.java b/enterprise/ha/src/test/java/org/neo4j/kernel/ha/cluster/SwitchToSlaveCopyThenBranchTest.java index ba6554bf928aa..89480b12d67cd 100644 --- a/enterprise/ha/src/test/java/org/neo4j/kernel/ha/cluster/SwitchToSlaveCopyThenBranchTest.java +++ b/enterprise/ha/src/test/java/org/neo4j/kernel/ha/cluster/SwitchToSlaveCopyThenBranchTest.java @@ -333,7 +333,6 @@ private SwitchToSlaveCopyThenBranch newSwitchToSlaveSpy() throws Exception FileSystemAbstraction fileSystemAbstraction = mock( FileSystemAbstraction.class ); when( fileSystemAbstraction.streamFilesRecursive( any( File.class ) ) ) .thenReturn( mockStream ); - when( pageCacheMock.getCachedFileSystem() ).thenReturn( fileSystemAbstraction ); return newSwitchToSlaveSpy( pageCacheMock, storeCopyClient ); } diff --git a/enterprise/kernel/src/test/java/org/neo4j/kernel/impl/store/format/RecordFormatSelectorTest.java b/enterprise/kernel/src/test/java/org/neo4j/kernel/impl/store/format/RecordFormatSelectorTest.java index 3b9694e41ed9e..7ec69d49782af 100644 --- a/enterprise/kernel/src/test/java/org/neo4j/kernel/impl/store/format/RecordFormatSelectorTest.java +++ b/enterprise/kernel/src/test/java/org/neo4j/kernel/impl/store/format/RecordFormatSelectorTest.java @@ -152,7 +152,7 @@ public void selectForStoreWithValidStore() throws IOException @Test public void selectForStoreWithNoStore() { - assertNull( selectForStore( storeDir, getPageCache(), LOG ) ); + assertNull( selectForStore( storeDir, fs, getPageCache(), LOG ) ); } @Test @@ -160,10 +160,9 @@ public void selectForStoreWithThrowingPageCache() throws IOException { createNeoStoreFile(); PageCache pageCache = mock( PageCache.class ); - when( pageCache.getCachedFileSystem() ).thenReturn( fs ); when( pageCache.pageSize() ).thenReturn( PageCache.PAGE_SIZE ); when( pageCache.map( any(), anyInt(), any() ) ).thenThrow( new IOException( "No reading..." ) ); - assertNull( selectForStore( storeDir, pageCache, LOG ) ); + assertNull( selectForStore( storeDir, fs, pageCache, LOG ) ); verify( pageCache ).map( any(), anyInt(), any() ); } @@ -172,7 +171,7 @@ public void selectForStoreWithInvalidStoreVersion() throws IOException { PageCache pageCache = getPageCache(); prepareNeoStoreFile( "v9.Z.9", pageCache ); - assertNull( selectForStore( storeDir, getPageCache(), LOG ) ); + assertNull( selectForStore( storeDir, fs, getPageCache(), LOG ) ); } @Test @@ -183,7 +182,7 @@ public void selectForStoreOrConfigWithSameStandardConfiguredAndStoredFormat() th Config config = config( Standard.LATEST_NAME ); - assertSame( Standard.LATEST_RECORD_FORMATS, selectForStoreOrConfig( config, storeDir, pageCache, LOG ) ); + assertSame( Standard.LATEST_RECORD_FORMATS, selectForStoreOrConfig( config, storeDir, fs, pageCache, LOG ) ); } @Test @@ -194,7 +193,7 @@ public void selectForStoreOrConfigWithSameHighLimitConfiguredAndStoredFormat() t Config config = config( HighLimit.NAME ); - assertSame( HighLimit.RECORD_FORMATS, selectForStoreOrConfig( config, storeDir, pageCache, LOG ) ); + assertSame( HighLimit.RECORD_FORMATS, selectForStoreOrConfig( config, storeDir, fs, pageCache, LOG ) ); } @Test @@ -207,7 +206,7 @@ public void selectForStoreOrConfigWithDifferentlyConfiguredAndStoredFormat() thr try { - selectForStoreOrConfig( config, storeDir, pageCache, LOG ); + selectForStoreOrConfig( config, storeDir, fs, pageCache, LOG ); fail( "Exception expected" ); } catch ( Exception e ) @@ -224,7 +223,7 @@ public void selectForStoreOrConfigWithOnlyStandardStoredFormat() throws IOExcept Config config = Config.defaults(); - assertSame( Standard.LATEST_RECORD_FORMATS, selectForStoreOrConfig( config, storeDir, pageCache, LOG ) ); + assertSame( Standard.LATEST_RECORD_FORMATS, selectForStoreOrConfig( config, storeDir, fs, pageCache, LOG ) ); } @Test @@ -235,7 +234,7 @@ public void selectForStoreOrConfigWithOnlyHighLimitStoredFormat() throws IOExcep Config config = Config.defaults(); - assertSame( HighLimit.RECORD_FORMATS, selectForStoreOrConfig( config, storeDir, pageCache, LOG ) ); + assertSame( HighLimit.RECORD_FORMATS, selectForStoreOrConfig( config, storeDir, fs, pageCache, LOG ) ); } @Test @@ -245,7 +244,7 @@ public void selectForStoreOrConfigWithOnlyStandardConfiguredFormat() Config config = config( Standard.LATEST_NAME ); - assertSame( Standard.LATEST_RECORD_FORMATS, selectForStoreOrConfig( config, storeDir, pageCache, LOG ) ); + assertSame( Standard.LATEST_RECORD_FORMATS, selectForStoreOrConfig( config, storeDir, fs, pageCache, LOG ) ); } @Test @@ -255,7 +254,7 @@ public void selectForStoreOrConfigWithOnlyHighLimitConfiguredFormat() Config config = config( HighLimit.NAME ); - assertSame( HighLimit.RECORD_FORMATS, selectForStoreOrConfig( config, storeDir, pageCache, LOG ) ); + assertSame( HighLimit.RECORD_FORMATS, selectForStoreOrConfig( config, storeDir, fs, pageCache, LOG ) ); } @Test @@ -267,7 +266,7 @@ public void selectForStoreOrConfigWithWrongConfiguredFormat() try { - selectForStoreOrConfig( config, storeDir, pageCache, LOG ); + selectForStoreOrConfig( config, storeDir, fs, pageCache, LOG ); } catch ( Exception e ) { @@ -278,21 +277,21 @@ public void selectForStoreOrConfigWithWrongConfiguredFormat() @Test public void selectForStoreOrConfigWithoutConfiguredAndStoredFormats() { - assertSame( defaultFormat(), selectForStoreOrConfig( Config.defaults(), storeDir, getPageCache(), LOG ) ); + assertSame( defaultFormat(), selectForStoreOrConfig( Config.defaults(), storeDir, fs, getPageCache(), LOG ) ); } @Test public void selectNewestFormatWithConfiguredStandardFormat() { assertSame( Standard.LATEST_RECORD_FORMATS, - selectNewestFormat( config( Standard.LATEST_NAME ), storeDir, getPageCache(), LOG ) ); + selectNewestFormat( config( Standard.LATEST_NAME ), storeDir, fs, getPageCache(), LOG ) ); } @Test public void selectNewestFormatWithConfiguredHighLimitFormat() { assertSame( HighLimit.RECORD_FORMATS, - selectNewestFormat( config( HighLimit.NAME ), storeDir, getPageCache(), LOG ) ); + selectNewestFormat( config( HighLimit.NAME ), storeDir, fs, getPageCache(), LOG ) ); } @Test @@ -300,7 +299,7 @@ public void selectNewestFormatWithWrongConfiguredFormat() { try { - selectNewestFormat( config( "unknown_format" ), storeDir, getPageCache(), LOG ); + selectNewestFormat( config( "unknown_format" ), storeDir, fs, getPageCache(), LOG ); } catch ( Exception e ) { @@ -311,7 +310,7 @@ public void selectNewestFormatWithWrongConfiguredFormat() @Test public void selectNewestFormatWithoutConfigAndStore() { - assertSame( defaultFormat(), selectNewestFormat( Config.defaults(), storeDir, getPageCache(), LOG ) ); + assertSame( defaultFormat(), selectNewestFormat( Config.defaults(), storeDir, fs, getPageCache(), LOG ) ); } @Test @@ -322,7 +321,7 @@ public void selectNewestFormatForExistingStandardStore() throws IOException Config config = Config.defaults(); - assertSame( Standard.LATEST_RECORD_FORMATS, selectNewestFormat( config, storeDir, getPageCache(), LOG ) ); + assertSame( Standard.LATEST_RECORD_FORMATS, selectNewestFormat( config, storeDir, fs, getPageCache(), LOG ) ); } @Test @@ -333,7 +332,7 @@ public void selectNewestFormatForExistingHighLimitStore() throws IOException Config config = Config.defaults(); - assertSame( HighLimit.RECORD_FORMATS, selectNewestFormat( config, storeDir, getPageCache(), LOG ) ); + assertSame( HighLimit.RECORD_FORMATS, selectNewestFormat( config, storeDir, fs, getPageCache(), LOG ) ); } @Test @@ -344,7 +343,7 @@ public void selectNewestFormatForExistingStoreWithLegacyFormat() throws IOExcept Config config = Config.defaults(); - assertSame( defaultFormat(), selectNewestFormat( config, storeDir, getPageCache(), LOG ) ); + assertSame( defaultFormat(), selectNewestFormat( config, storeDir, fs, getPageCache(), LOG ) ); } @Test @@ -374,7 +373,7 @@ private PageCache getPageCache() private void verifySelectForStore( PageCache pageCache, RecordFormats format ) throws IOException { prepareNeoStoreFile( format.storeVersion(), pageCache ); - assertSame( format, selectForStore( storeDir, pageCache, LOG ) ); + assertSame( format, selectForStore( storeDir, fs, pageCache, LOG ) ); } private File prepareNeoStoreFile( String storeVersion, PageCache pageCache ) throws IOException diff --git a/enterprise/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/StoreMigrationIT.java b/enterprise/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/StoreMigrationIT.java index dc3f63b7ac9eb..808560f004411 100644 --- a/enterprise/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/StoreMigrationIT.java +++ b/enterprise/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/StoreMigrationIT.java @@ -304,7 +304,6 @@ protected Stream stream( Iterable iterable ) return StreamSupport.stream( iterable.spliterator(), false ); } - //This method is overridden by a blockdevice test. protected ConsistencyCheckService.Result runConsistencyChecker( File db, FileSystemAbstraction fs, ConsistencyCheckService consistencyCheckService, String storeVersion ) throws ConsistencyCheckIncompleteException @@ -314,7 +313,6 @@ protected ConsistencyCheckService.Result runConsistencyChecker( File db, FileSys NullLogProvider.getInstance(), fs, false ); } - //This method is overridden by a blockdevice test. protected GraphDatabaseService getGraphDatabaseService( File db, String storeVersion ) { return new EnterpriseGraphDatabaseFactory().newEmbeddedDatabaseBuilder( db ) diff --git a/enterprise/neo4j-enterprise/src/test/java/org/neo4j/upgrade/RecordFormatsMigrationIT.java b/enterprise/neo4j-enterprise/src/test/java/org/neo4j/upgrade/RecordFormatsMigrationIT.java index 77e2a14ea8a0b..e4be1f74081b0 100644 --- a/enterprise/neo4j-enterprise/src/test/java/org/neo4j/upgrade/RecordFormatsMigrationIT.java +++ b/enterprise/neo4j-enterprise/src/test/java/org/neo4j/upgrade/RecordFormatsMigrationIT.java @@ -157,7 +157,7 @@ private void assertStoreFormat( RecordFormats expected ) try ( PageCache pageCache = ConfigurableStandalonePageCacheFactory.createPageCache( fileSystemRule.get(), config ) ) { RecordFormats actual = RecordFormatSelector.selectForStoreOrConfig( config, testDirectory.graphDbDir(), - pageCache, NullLogProvider.getInstance() ); + fileSystemRule, pageCache, NullLogProvider.getInstance() ); assertNotNull( actual ); assertEquals( expected.storeVersion(), actual.storeVersion() ); }