From 4d9a3717330c760bceeab5550388f03d00b9472f Mon Sep 17 00:00:00 2001 From: Mikhaylo Demianenko Date: Thu, 14 Apr 2016 14:00:28 +0200 Subject: [PATCH] Simplify page cache adversarial tests, reorganise page cache harness tests Remove BootClassPathRunner infrustructure of page cache adversarial tests since those where extremelly flaky under IBM JDK. Switch those tests to inject and use adversarial file system or adversarial channel instead of custom file dispatcher. Extract page cache harness into separate tests. --- .../io/fs/DefaultFileSystemAbstraction.java | 7 +- ...alChannelDefaultFileSystemAbstraction.java | 54 +++++ .../fs/AdversarialFileChannel.java | 44 +--- .../fs/AdversarialFileDispatcherFactory.java | 94 -------- .../fs/AdversarialFileSystemAbstraction.java | 5 +- .../org/neo4j/io/pagecache/PageCacheTest.java | 150 +----------- .../io/pagecache/PageCacheTestSupport.java | 13 - ...nPageCacheHarnessWithRealFileSystemIT.java | 40 +++ .../harness/MunninPageCacheHarnessTest.java | 32 +++ .../harness/PageCacheHarnessTest.java | 202 ++++++++++++++++ ...eSwapperWithAdversarialFileChannelIT.java} | 25 +- .../impl/muninn/MuninnPageCacheFixture.java | 2 +- ...eCacheSlowTestWithAdversarialChannel.java} | 26 +- ...eCacheWithAdversarialFileDispatcherIT.java | 57 ----- .../BootClassPathRunner.java | 147 ----------- .../DelegatingRemoteRunNotifier.java | 103 -------- .../DelegatingRunNotifier.java | 171 ------------- .../RemoteRunNotifier.java | 54 ----- .../test/bootclasspathrunner/RmiServer.java | 89 ------- .../sun/nio/ch/AccessibleFileDisptacher.java | 46 ---- .../sun/nio/ch/DelegateFileDispatcher.java | 228 ------------------ 21 files changed, 361 insertions(+), 1228 deletions(-) create mode 100644 community/io/src/test/java/org/neo4j/adversaries/fs/AdversarialChannelDefaultFileSystemAbstraction.java delete mode 100644 community/io/src/test/java/org/neo4j/adversaries/fs/AdversarialFileDispatcherFactory.java create mode 100644 community/io/src/test/java/org/neo4j/io/pagecache/harness/MuninnPageCacheHarnessWithRealFileSystemIT.java create mode 100644 community/io/src/test/java/org/neo4j/io/pagecache/harness/MunninPageCacheHarnessTest.java create mode 100644 community/io/src/test/java/org/neo4j/io/pagecache/harness/PageCacheHarnessTest.java rename community/io/src/test/java/org/neo4j/io/pagecache/impl/{SingleFilePageSwapperWithAdversarialFileDispatcherIT.java => SingleFilePageSwapperWithAdversarialFileChannelIT.java} (51%) rename community/io/src/test/java/org/neo4j/io/pagecache/impl/muninn/{MuninnPageCacheSlowTestWithAdversarialFileDispatcherIT.java => MuninnPageCacheSlowTestWithAdversarialChannel.java} (52%) delete mode 100644 community/io/src/test/java/org/neo4j/io/pagecache/impl/muninn/MuninnPageCacheWithAdversarialFileDispatcherIT.java delete mode 100644 community/io/src/test/java/org/neo4j/test/bootclasspathrunner/BootClassPathRunner.java delete mode 100644 community/io/src/test/java/org/neo4j/test/bootclasspathrunner/DelegatingRemoteRunNotifier.java delete mode 100644 community/io/src/test/java/org/neo4j/test/bootclasspathrunner/DelegatingRunNotifier.java delete mode 100644 community/io/src/test/java/org/neo4j/test/bootclasspathrunner/RemoteRunNotifier.java delete mode 100644 community/io/src/test/java/org/neo4j/test/bootclasspathrunner/RmiServer.java delete mode 100644 community/unsafe/src/test/java/sun/nio/ch/AccessibleFileDisptacher.java delete mode 100644 community/unsafe/src/test/java/sun/nio/ch/DelegateFileDispatcher.java diff --git a/community/io/src/main/java/org/neo4j/io/fs/DefaultFileSystemAbstraction.java b/community/io/src/main/java/org/neo4j/io/fs/DefaultFileSystemAbstraction.java index d5f612f41afb3..b8f1736ebd8c1 100644 --- a/community/io/src/main/java/org/neo4j/io/fs/DefaultFileSystemAbstraction.java +++ b/community/io/src/main/java/org/neo4j/io/fs/DefaultFileSystemAbstraction.java @@ -51,7 +51,7 @@ public StoreFileChannel open( File fileName, String mode ) throws IOException { // Returning only the channel is ok, because the channel, when close()d will close its parent File. FileChannel channel = new RandomAccessFile( fileName, mode ).getChannel(); - return new StoreFileChannel( channel ); + return getStoreFileChannel( channel ); } @Override @@ -194,4 +194,9 @@ public void truncate( File path, long size ) throws IOException { FileUtils.truncateFile( path, size ); } + + protected StoreFileChannel getStoreFileChannel( FileChannel channel ) + { + return new StoreFileChannel( channel ); + } } diff --git a/community/io/src/test/java/org/neo4j/adversaries/fs/AdversarialChannelDefaultFileSystemAbstraction.java b/community/io/src/test/java/org/neo4j/adversaries/fs/AdversarialChannelDefaultFileSystemAbstraction.java new file mode 100644 index 0000000000000..7f551f2120370 --- /dev/null +++ b/community/io/src/test/java/org/neo4j/adversaries/fs/AdversarialChannelDefaultFileSystemAbstraction.java @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2002-2016 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.adversaries.fs; + +import java.nio.channels.FileChannel; + +import org.neo4j.adversaries.RandomAdversary; +import org.neo4j.io.fs.DefaultFileSystemAbstraction; +import org.neo4j.io.fs.StoreFileChannel; + +/** + * File system abstraction that behaves exactly like {@link DefaultFileSystemAbstraction} except instead of + * default {@link FileChannel} implementation {@link AdversarialFileChannel} will be used. + * + * This abstraction should be used in cases when it's desirable to have default file system implementation + * and only verify handling of inconsistent channel operations. + * Otherwise consider {@link AdversarialFileSystemAbstraction} since it should produce more failure cases. + */ +public class AdversarialChannelDefaultFileSystemAbstraction extends DefaultFileSystemAbstraction +{ + private final RandomAdversary adversary; + + public AdversarialChannelDefaultFileSystemAbstraction() + { + this( new RandomAdversary( 0.5, 0.0, 0.0 ) ); + } + + public AdversarialChannelDefaultFileSystemAbstraction( RandomAdversary adversary ) + { + this.adversary = adversary; + } + + protected StoreFileChannel getStoreFileChannel( FileChannel channel ) + { + return AdversarialFileChannel.wrap( super.getStoreFileChannel( channel ), adversary ); + } +} diff --git a/community/io/src/test/java/org/neo4j/adversaries/fs/AdversarialFileChannel.java b/community/io/src/test/java/org/neo4j/adversaries/fs/AdversarialFileChannel.java index 22969a1a0158c..0b03d467a352e 100644 --- a/community/io/src/test/java/org/neo4j/adversaries/fs/AdversarialFileChannel.java +++ b/community/io/src/test/java/org/neo4j/adversaries/fs/AdversarialFileChannel.java @@ -19,56 +19,28 @@ */ package org.neo4j.adversaries.fs; -import sun.nio.ch.FileChannelImpl; - import java.io.IOException; -import java.lang.reflect.Field; import java.nio.ByteBuffer; -import java.nio.channels.FileChannel; import java.nio.channels.FileLock; import org.neo4j.adversaries.Adversary; import org.neo4j.io.fs.StoreChannel; import org.neo4j.io.fs.StoreFileChannel; -import org.neo4j.io.fs.StoreFileChannelUnwrapper; - -import static org.neo4j.adversaries.fs.AdversarialFileDispatcherFactory.makeFileDispatcherAdversarial; @SuppressWarnings( "unchecked" ) -public class AdversarialFileChannel implements StoreChannel +public class AdversarialFileChannel extends StoreFileChannel { - public static volatile boolean useAdversarialFileDispatcherHack; - private final StoreChannel delegate; private final Adversary adversary; - public static StoreChannel wrap( StoreChannel channel, Adversary adversary ) + public static StoreFileChannel wrap( StoreFileChannel channel, Adversary adversary ) { - if ( useAdversarialFileDispatcherHack && channel.getClass() == StoreFileChannel.class ) - { - FileChannel innerChannel = StoreFileChannelUnwrapper.unwrap( channel ); - if ( innerChannel.getClass() == FileChannelImpl.class ) - { - FileChannelImpl channelImpl = (FileChannelImpl) innerChannel; - try - { - Field nd = FileChannelImpl.class.getDeclaredField( "nd" ); - nd.setAccessible( true ); - Object fileDispatcher = nd.get( channelImpl ); - nd.set( channelImpl, makeFileDispatcherAdversarial( fileDispatcher, adversary ) ); - return new StoreFileChannel( innerChannel ); - } - catch ( Exception e ) - { - e.printStackTrace(); - } - } - } return new AdversarialFileChannel( channel, adversary ); } - private AdversarialFileChannel( StoreChannel channel, Adversary adversary ) + private AdversarialFileChannel( StoreFileChannel channel, Adversary adversary ) { + super( channel ); this.delegate = channel; this.adversary = adversary; } @@ -130,17 +102,17 @@ public long write( ByteBuffer[] srcs, int offset, int length ) throws IOExceptio } @Override - public StoreChannel truncate( long size ) throws IOException + public StoreFileChannel truncate( long size ) throws IOException { adversary.injectFailure( IOException.class ); - return delegate.truncate( size ); + return (StoreFileChannel) delegate.truncate( size ); } @Override - public StoreChannel position( long newPosition ) throws IOException + public StoreFileChannel position( long newPosition ) throws IOException { adversary.injectFailure( IOException.class ); - return delegate.position( newPosition ); + return (StoreFileChannel) delegate.position( newPosition ); } @Override diff --git a/community/io/src/test/java/org/neo4j/adversaries/fs/AdversarialFileDispatcherFactory.java b/community/io/src/test/java/org/neo4j/adversaries/fs/AdversarialFileDispatcherFactory.java deleted file mode 100644 index 4563d94765b84..0000000000000 --- a/community/io/src/test/java/org/neo4j/adversaries/fs/AdversarialFileDispatcherFactory.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Copyright (c) 2002-2016 "Neo Technology," - * Network Engine for Objects in Lund AB [http://neotechnology.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package org.neo4j.adversaries.fs; - -import sun.nio.ch.DelegateFileDispatcher; - -import java.io.FileDescriptor; -import java.io.IOException; -import java.util.concurrent.ThreadLocalRandom; - -import org.neo4j.adversaries.Adversary; - -@SuppressWarnings( "unchecked" ) -public class AdversarialFileDispatcherFactory -{ - public static Object makeFileDispatcherAdversarial( - final Object delegateFileDispatcher, - final Adversary adversary ) throws Exception - { - return new DelegateFileDispatcher( delegateFileDispatcher ) - { - private int mischievousLength( int len ) - { - if ( adversary.injectFailureOrMischief( IOException.class ) ) - { - // We cannot pass zero lengths, because they are checked for and - // circumvented in higher-level NIO code. - len = len == 1? 1 : ThreadLocalRandom.current().nextInt( 1, len ); - } - return len; - } - - @Override - public long readv( FileDescriptor fd, long address, int len ) throws IOException - { - return super.readv( fd, address, mischievousLength( len ) ); - } - - @Override - public int read( FileDescriptor fd, long address, int len ) throws IOException - { - return super.read( fd, address, mischievousLength( len ) ); - } - - @Override - public int pwrite( FileDescriptor fd, long address, int len, long position ) throws IOException - { - return super.pwrite( fd, address, mischievousLength( len ), position ); - } - - @Override - public int truncate( FileDescriptor fd, long size ) throws IOException - { - adversary.injectFailure( IOException.class ); - return super.truncate( fd, size ); - } - - @Override - public int pread( FileDescriptor fd, long address, int len, long position ) throws IOException - { - return super.pread( fd, address, mischievousLength( len ), position ); - } - - @Override - public long writev( FileDescriptor fd, long address, int len ) throws IOException - { - return super.writev( fd, address, mischievousLength( len ) ); - } - - @Override - public int write( FileDescriptor fd, long address, int len ) throws IOException - { - return super.write( fd, address, mischievousLength( len ) ); - } - }; - } -} diff --git a/community/io/src/test/java/org/neo4j/adversaries/fs/AdversarialFileSystemAbstraction.java b/community/io/src/test/java/org/neo4j/adversaries/fs/AdversarialFileSystemAbstraction.java index d9acd05d62a9d..5846026f9cbe2 100644 --- a/community/io/src/test/java/org/neo4j/adversaries/fs/AdversarialFileSystemAbstraction.java +++ b/community/io/src/test/java/org/neo4j/adversaries/fs/AdversarialFileSystemAbstraction.java @@ -40,6 +40,7 @@ import org.neo4j.io.fs.DefaultFileSystemAbstraction; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.fs.StoreChannel; +import org.neo4j.io.fs.StoreFileChannel; /** * Used by the robustness suite to check for partial failures. @@ -64,7 +65,7 @@ public AdversarialFileSystemAbstraction( Adversary adversary, FileSystemAbstract public StoreChannel open( File fileName, String mode ) throws IOException { adversary.injectFailure( FileNotFoundException.class, IOException.class, SecurityException.class ); - return AdversarialFileChannel.wrap( delegate.open( fileName, mode ), adversary ); + return AdversarialFileChannel.wrap( (StoreFileChannel) delegate.open( fileName, mode ), adversary ); } public boolean renameFile( File from, File to ) throws IOException @@ -82,7 +83,7 @@ public OutputStream openAsOutputStream( File fileName, boolean append ) throws I public StoreChannel create( File fileName ) throws IOException { adversary.injectFailure( FileNotFoundException.class, IOException.class, SecurityException.class ); - return AdversarialFileChannel.wrap( delegate.create( fileName ), adversary ); + return AdversarialFileChannel.wrap( (StoreFileChannel) delegate.create( fileName ), adversary ); } public boolean mkdir( File fileName ) diff --git a/community/io/src/test/java/org/neo4j/io/pagecache/PageCacheTest.java b/community/io/src/test/java/org/neo4j/io/pagecache/PageCacheTest.java index 0f0a7457bb607..6b64e5dbb6bbb 100644 --- a/community/io/src/test/java/org/neo4j/io/pagecache/PageCacheTest.java +++ b/community/io/src/test/java/org/neo4j/io/pagecache/PageCacheTest.java @@ -79,6 +79,7 @@ import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import static org.junit.Assume.assumeTrue; import static org.neo4j.io.pagecache.PagedFile.PF_NO_FAULT; import static org.neo4j.io.pagecache.PagedFile.PF_NO_GROW; import static org.neo4j.io.pagecache.PagedFile.PF_SHARED_READ_LOCK; @@ -1565,24 +1566,6 @@ public void pagesAddedWithNextWithPageIdMustBeAccessibleWithNoGrowSpecified() th pagedFile.close(); } - @RepeatRule.Repeat( times = 10 ) - @Test( timeout = SEMI_LONG_TIMEOUT_MILLIS ) - public void readsAndWritesMustBeMutuallyConsistent() throws Exception - { - int filePageCount = 100; - RandomPageCacheTestHarness harness = new RandomPageCacheTestHarness(); - harness.disableCommands( FlushCache, FlushFile, MapFile, UnmapFile ); - harness.setCommandProbabilityFactor( ReadRecord, 0.5 ); - harness.setCommandProbabilityFactor( WriteRecord, 0.5 ); - harness.setConcurrencyLevel( 8 ); - harness.setFilePageCount( filePageCount ); - harness.setInitialMappedFiles( 1 ); - harness.setCachePageSize( pageCachePageSize ); - harness.setFilePageSize( pageCachePageSize ); - harness.setVerification( filesAreCorrectlyWrittenVerification( new StandardRecordFormat(), filePageCount ) ); - harness.run( SEMI_LONG_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS ); - } - @Test( timeout = SHORT_TIMEOUT_MILLIS ) public void writesOfDifferentUnitsMustHaveCorrectEndianess() throws Exception { @@ -3174,137 +3157,6 @@ public void optimisticReadLockMustFaultOnRetryIfPageHasBeenEvicted() throws Exce pagedFileB.close(); } - @Test( timeout = SEMI_LONG_TIMEOUT_MILLIS ) - public void concurrentPageFaultingMustNotPutInterleavedDataIntoPages() throws Exception - { - final int filePageCount = 11; - final RecordFormat recordFormat = new PageCountRecordFormat(); - RandomPageCacheTestHarness harness = new RandomPageCacheTestHarness(); - harness.setConcurrencyLevel( 11 ); - harness.setUseAdversarialIO( false ); - harness.setCachePageCount( 3 ); - harness.setCachePageSize( pageCachePageSize ); - harness.setFilePageCount( filePageCount ); - harness.setFilePageSize( pageCachePageSize ); - harness.setInitialMappedFiles( 1 ); - harness.setCommandCount( 10000 ); - harness.setRecordFormat( recordFormat ); - harness.setFileSystem( fs ); - harness.disableCommands( FlushCache, FlushFile, MapFile, UnmapFile, WriteRecord, WriteMulti ); - harness.setPreparation( ( pageCache1, fs1, filesTouched ) -> { - File file = filesTouched.iterator().next(); - try ( PagedFile pf = pageCache1.map( file, pageCachePageSize ); - PageCursor cursor = pf.io( 0, PF_SHARED_WRITE_LOCK ) ) - { - for ( int pageId = 0; pageId < filePageCount; pageId++ ) - { - cursor.next(); - recordFormat.fillWithRecords( cursor ); - } - } - } ); - - harness.run( SEMI_LONG_TIMEOUT_MILLIS, MILLISECONDS ); - } - - @Test( timeout = SEMI_LONG_TIMEOUT_MILLIS ) - public void concurrentFlushingMustNotPutInterleavedDataIntoFile() throws Exception - { - final RecordFormat recordFormat = new StandardRecordFormat(); - final int filePageCount = 2_000; - RandomPageCacheTestHarness harness = new RandomPageCacheTestHarness(); - harness.setConcurrencyLevel( 16 ); - harness.setUseAdversarialIO( false ); - harness.setCachePageCount( filePageCount / 2 ); - harness.setFilePageCount( filePageCount ); - harness.setCachePageSize( pageCachePageSize ); - harness.setFilePageSize( pageCachePageSize ); - harness.setInitialMappedFiles( 3 ); - harness.setCommandCount( 15_000 ); - harness.setFileSystem( fs ); - harness.disableCommands( MapFile, UnmapFile, ReadRecord, ReadMulti ); - harness.setVerification( filesAreCorrectlyWrittenVerification( recordFormat, filePageCount ) ); - - harness.run( SEMI_LONG_TIMEOUT_MILLIS, MILLISECONDS ); - } - - @Test( timeout = SEMI_LONG_TIMEOUT_MILLIS ) - public void concurrentFlushingWithMischiefMustNotPutInterleavedDataIntoFile() throws Exception - { - final RecordFormat recordFormat = new StandardRecordFormat(); - final int filePageCount = 2_000; - RandomPageCacheTestHarness harness = new RandomPageCacheTestHarness(); - harness.setConcurrencyLevel( 16 ); - harness.setUseAdversarialIO( true ); - harness.setMischiefRate( 0.5 ); - harness.setFailureRate( 0.0 ); - harness.setErrorRate( 0.0 ); - harness.setCachePageCount( filePageCount / 2 ); - harness.setFilePageCount( filePageCount ); - harness.setCachePageSize( pageCachePageSize ); - harness.setFilePageSize( pageCachePageSize ); - harness.setInitialMappedFiles( 3 ); - harness.setCommandCount( 15_000 ); - harness.setFileSystem( fs ); - harness.disableCommands( MapFile, UnmapFile, ReadRecord, ReadMulti ); - harness.setVerification( filesAreCorrectlyWrittenVerification( recordFormat, filePageCount ) ); - - harness.run( SEMI_LONG_TIMEOUT_MILLIS, MILLISECONDS ); - } - - @Test( timeout = SEMI_LONG_TIMEOUT_MILLIS ) - public void concurrentFlushingWithFailuresMustNotPutInterleavedDataIntoFile() throws Exception - { - final RecordFormat recordFormat = new StandardRecordFormat(); - final int filePageCount = 20_000; - RandomPageCacheTestHarness harness = new RandomPageCacheTestHarness(); - harness.setConcurrencyLevel( 16 ); - harness.setUseAdversarialIO( true ); - harness.setMischiefRate( 0.0 ); - harness.setFailureRate( 0.5 ); - harness.setErrorRate( 0.0 ); - harness.setCachePageCount( filePageCount / 2 ); - harness.setFilePageCount( filePageCount ); - harness.setCachePageSize( pageCachePageSize ); - harness.setFilePageSize( pageCachePageSize ); - harness.setInitialMappedFiles( 3 ); - harness.setCommandCount( 150_000 ); - harness.setFileSystem( fs ); - harness.disableCommands( MapFile, UnmapFile, ReadRecord, ReadMulti ); - harness.setVerification( filesAreCorrectlyWrittenVerification( recordFormat, filePageCount ) ); - - harness.run( SEMI_LONG_TIMEOUT_MILLIS, MILLISECONDS ); - } - - private Phase filesAreCorrectlyWrittenVerification( final RecordFormat recordFormat, final int filePageCount ) - { - return ( pageCache1, fs1, filesTouched ) -> { - for ( File file : filesTouched ) - { - try ( PagedFile pf = pageCache1.map( file, pageCachePageSize ); - PageCursor cursor = pf.io( 0, PF_SHARED_READ_LOCK ) ) - { - for ( int pageId = 0; pageId < filePageCount && cursor.next(); pageId++ ) - { - try - { - recordFormat.assertRecordsWrittenCorrectly( cursor ); - } - catch ( Throwable th ) - { - th.addSuppressed( new Exception( "pageId = " + pageId ) ); - throw th; - } - } - } - try ( StoreChannel channel = fs1.open( file, "r" ) ) - { - recordFormat.assertRecordsWrittenCorrectly( file, channel ); - } - } - }; - } - @Test( timeout = SEMI_LONG_TIMEOUT_MILLIS ) public void backgroundThreadsMustGracefullyShutDown() throws Exception { diff --git a/community/io/src/test/java/org/neo4j/io/pagecache/PageCacheTestSupport.java b/community/io/src/test/java/org/neo4j/io/pagecache/PageCacheTestSupport.java index 3ffaf89728d0b..51b21016b38da 100644 --- a/community/io/src/test/java/org/neo4j/io/pagecache/PageCacheTestSupport.java +++ b/community/io/src/test/java/org/neo4j/io/pagecache/PageCacheTestSupport.java @@ -284,19 +284,6 @@ protected void verifyRecordsInFile( File file, int recordCount ) throws IOExcept }; } - /** - * We implement 'assumeTrue' ourselves because JUnit insist on adding hamcrest matchers to the - * AssumptionViolatedException instances it throws. This is a problem because those matchers are not serializable, - * so they cannot be used together with the BootClassPathRunner, because it uses RMI under the hood. - */ - protected void assumeTrue( String description, boolean test ) - { - if ( !test ) - { - throw new AssumptionViolatedException( description ); - } - } - public static abstract class Fixture { public abstract T createPageCache( diff --git a/community/io/src/test/java/org/neo4j/io/pagecache/harness/MuninnPageCacheHarnessWithRealFileSystemIT.java b/community/io/src/test/java/org/neo4j/io/pagecache/harness/MuninnPageCacheHarnessWithRealFileSystemIT.java new file mode 100644 index 0000000000000..65bb129efe280 --- /dev/null +++ b/community/io/src/test/java/org/neo4j/io/pagecache/harness/MuninnPageCacheHarnessWithRealFileSystemIT.java @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2002-2016 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.io.pagecache.harness; + +import org.junit.Rule; + +import org.neo4j.io.fs.DefaultFileSystemAbstraction; +import org.neo4j.io.pagecache.impl.muninn.MuninnPageCache; +import org.neo4j.test.TargetDirectory; + +public class MuninnPageCacheHarnessWithRealFileSystemIT extends MunninPageCacheHarnessTest +{ + @Rule + public TargetDirectory.TestDirectory directory = TargetDirectory.testDirForTest( getClass() ); + + @Override + protected Fixture createFixture() + { + return super.createFixture() + .withFileSystemAbstraction( DefaultFileSystemAbstraction::new ) + .withFileConstructor( pathname -> directory.file( pathname ) ); + } +} diff --git a/community/io/src/test/java/org/neo4j/io/pagecache/harness/MunninPageCacheHarnessTest.java b/community/io/src/test/java/org/neo4j/io/pagecache/harness/MunninPageCacheHarnessTest.java new file mode 100644 index 0000000000000..787a022b1f7ac --- /dev/null +++ b/community/io/src/test/java/org/neo4j/io/pagecache/harness/MunninPageCacheHarnessTest.java @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2002-2016 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.io.pagecache.harness; + +import org.neo4j.io.pagecache.impl.muninn.MuninnPageCache; +import org.neo4j.io.pagecache.impl.muninn.MuninnPageCacheFixture; + +public class MunninPageCacheHarnessTest extends PageCacheHarnessTest +{ + @Override + protected Fixture createFixture() + { + return new MuninnPageCacheFixture(); + } +} diff --git a/community/io/src/test/java/org/neo4j/io/pagecache/harness/PageCacheHarnessTest.java b/community/io/src/test/java/org/neo4j/io/pagecache/harness/PageCacheHarnessTest.java new file mode 100644 index 0000000000000..8719a8f147e5c --- /dev/null +++ b/community/io/src/test/java/org/neo4j/io/pagecache/harness/PageCacheHarnessTest.java @@ -0,0 +1,202 @@ +/* + * Copyright (c) 2002-2016 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.io.pagecache.harness; + +import org.junit.Test; + +import java.io.File; +import java.util.concurrent.TimeUnit; + +import org.neo4j.io.fs.StoreChannel; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.PageCacheTestSupport; +import org.neo4j.io.pagecache.PageCursor; +import org.neo4j.io.pagecache.PagedFile; +import org.neo4j.io.pagecache.randomharness.PageCountRecordFormat; +import org.neo4j.io.pagecache.randomharness.Phase; +import org.neo4j.io.pagecache.randomharness.RandomPageCacheTestHarness; +import org.neo4j.io.pagecache.randomharness.RecordFormat; +import org.neo4j.io.pagecache.randomharness.StandardRecordFormat; +import org.neo4j.test.RepeatRule; + +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static org.neo4j.io.pagecache.PagedFile.PF_SHARED_READ_LOCK; +import static org.neo4j.io.pagecache.PagedFile.PF_SHARED_WRITE_LOCK; +import static org.neo4j.io.pagecache.randomharness.Command.FlushCache; +import static org.neo4j.io.pagecache.randomharness.Command.FlushFile; +import static org.neo4j.io.pagecache.randomharness.Command.MapFile; +import static org.neo4j.io.pagecache.randomharness.Command.ReadMulti; +import static org.neo4j.io.pagecache.randomharness.Command.ReadRecord; +import static org.neo4j.io.pagecache.randomharness.Command.UnmapFile; +import static org.neo4j.io.pagecache.randomharness.Command.WriteMulti; +import static org.neo4j.io.pagecache.randomharness.Command.WriteRecord; + +abstract class PageCacheHarnessTest extends PageCacheTestSupport +{ + @RepeatRule.Repeat( times = 10 ) + @Test( timeout = SEMI_LONG_TIMEOUT_MILLIS ) + public void readsAndWritesMustBeMutuallyConsistent() throws Exception + { + int filePageCount = 100; + RandomPageCacheTestHarness harness = new RandomPageCacheTestHarness(); + harness.disableCommands( FlushCache, FlushFile, MapFile, UnmapFile ); + harness.setCommandProbabilityFactor( ReadRecord, 0.5 ); + harness.setCommandProbabilityFactor( WriteRecord, 0.5 ); + harness.setConcurrencyLevel( 8 ); + harness.setFilePageCount( filePageCount ); + harness.setInitialMappedFiles( 1 ); + harness.setCachePageSize( pageCachePageSize ); + harness.setFilePageSize( pageCachePageSize ); + harness.setVerification( filesAreCorrectlyWrittenVerification( new StandardRecordFormat(), filePageCount ) ); + harness.run( SEMI_LONG_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS ); + } + + @Test( timeout = SEMI_LONG_TIMEOUT_MILLIS ) + public void concurrentPageFaultingMustNotPutInterleavedDataIntoPages() throws Exception + { + final int filePageCount = 11; + final RecordFormat recordFormat = new PageCountRecordFormat(); + RandomPageCacheTestHarness harness = new RandomPageCacheTestHarness(); + harness.setConcurrencyLevel( 11 ); + harness.setUseAdversarialIO( false ); + harness.setCachePageCount( 3 ); + harness.setCachePageSize( pageCachePageSize ); + harness.setFilePageCount( filePageCount ); + harness.setFilePageSize( pageCachePageSize ); + harness.setInitialMappedFiles( 1 ); + harness.setCommandCount( 10000 ); + harness.setRecordFormat( recordFormat ); + harness.setFileSystem( fs ); + harness.disableCommands( FlushCache, FlushFile, MapFile, UnmapFile, WriteRecord, WriteMulti ); + harness.setPreparation( ( pageCache1, fs1, filesTouched ) -> { + File file = filesTouched.iterator().next(); + try ( PagedFile pf = pageCache1.map( file, pageCachePageSize ); + PageCursor cursor = pf.io( 0, PF_SHARED_WRITE_LOCK ) ) + { + for ( int pageId = 0; pageId < filePageCount; pageId++ ) + { + cursor.next(); + recordFormat.fillWithRecords( cursor ); + } + } + } ); + + harness.run( SEMI_LONG_TIMEOUT_MILLIS, MILLISECONDS ); + } + + @Test( timeout = SEMI_LONG_TIMEOUT_MILLIS ) + public void concurrentFlushingMustNotPutInterleavedDataIntoFile() throws Exception + { + final RecordFormat recordFormat = new StandardRecordFormat(); + final int filePageCount = 2_000; + RandomPageCacheTestHarness harness = new RandomPageCacheTestHarness(); + harness.setConcurrencyLevel( 16 ); + harness.setUseAdversarialIO( false ); + harness.setCachePageCount( filePageCount / 2 ); + harness.setFilePageCount( filePageCount ); + harness.setCachePageSize( pageCachePageSize ); + harness.setFilePageSize( pageCachePageSize ); + harness.setInitialMappedFiles( 3 ); + harness.setCommandCount( 15_000 ); + harness.setFileSystem( fs ); + harness.disableCommands( MapFile, UnmapFile, ReadRecord, ReadMulti ); + harness.setVerification( filesAreCorrectlyWrittenVerification( recordFormat, filePageCount ) ); + + harness.run( SEMI_LONG_TIMEOUT_MILLIS, MILLISECONDS ); + } + + @Test( timeout = SEMI_LONG_TIMEOUT_MILLIS ) + public void concurrentFlushingWithMischiefMustNotPutInterleavedDataIntoFile() throws Exception + { + final RecordFormat recordFormat = new StandardRecordFormat(); + final int filePageCount = 2_000; + RandomPageCacheTestHarness harness = new RandomPageCacheTestHarness(); + harness.setConcurrencyLevel( 16 ); + harness.setUseAdversarialIO( true ); + harness.setMischiefRate( 0.5 ); + harness.setFailureRate( 0.0 ); + harness.setErrorRate( 0.0 ); + harness.setCachePageCount( filePageCount / 2 ); + harness.setFilePageCount( filePageCount ); + harness.setCachePageSize( pageCachePageSize ); + harness.setFilePageSize( pageCachePageSize ); + harness.setInitialMappedFiles( 3 ); + harness.setCommandCount( 15_000 ); + harness.setFileSystem( fs ); + harness.disableCommands( MapFile, UnmapFile, ReadRecord, ReadMulti ); + harness.setVerification( filesAreCorrectlyWrittenVerification( recordFormat, filePageCount ) ); + + harness.run( SEMI_LONG_TIMEOUT_MILLIS, MILLISECONDS ); + } + + @Test( timeout = SEMI_LONG_TIMEOUT_MILLIS ) + public void concurrentFlushingWithFailuresMustNotPutInterleavedDataIntoFile() throws Exception + { + final RecordFormat recordFormat = new StandardRecordFormat(); + final int filePageCount = 20_000; + RandomPageCacheTestHarness harness = new RandomPageCacheTestHarness(); + harness.setConcurrencyLevel( 16 ); + harness.setUseAdversarialIO( true ); + harness.setMischiefRate( 0.0 ); + harness.setFailureRate( 0.5 ); + harness.setErrorRate( 0.0 ); + harness.setCachePageCount( filePageCount / 2 ); + harness.setFilePageCount( filePageCount ); + harness.setCachePageSize( pageCachePageSize ); + harness.setFilePageSize( pageCachePageSize ); + harness.setInitialMappedFiles( 3 ); + harness.setCommandCount( 150_000 ); + harness.setFileSystem( fs ); + harness.disableCommands( MapFile, UnmapFile, ReadRecord, ReadMulti ); + harness.setVerification( filesAreCorrectlyWrittenVerification( recordFormat, filePageCount ) ); + + harness.run( SEMI_LONG_TIMEOUT_MILLIS, MILLISECONDS ); + } + + private Phase filesAreCorrectlyWrittenVerification( final RecordFormat recordFormat, final int filePageCount ) + { + return ( pageCache1, fs1, filesTouched ) -> { + for ( File file : filesTouched ) + { + try ( PagedFile pf = pageCache1.map( file, pageCachePageSize ); + PageCursor cursor = pf.io( 0, PF_SHARED_READ_LOCK ) ) + { + for ( int pageId = 0; pageId < filePageCount && cursor.next(); pageId++ ) + { + try + { + recordFormat.assertRecordsWrittenCorrectly( cursor ); + } + catch ( Throwable th ) + { + th.addSuppressed( new Exception( "pageId = " + pageId ) ); + throw th; + } + } + } + try ( StoreChannel channel = fs1.open( file, "r" ) ) + { + recordFormat.assertRecordsWrittenCorrectly( file, channel ); + } + } + }; + } + +} diff --git a/community/io/src/test/java/org/neo4j/io/pagecache/impl/SingleFilePageSwapperWithAdversarialFileDispatcherIT.java b/community/io/src/test/java/org/neo4j/io/pagecache/impl/SingleFilePageSwapperWithAdversarialFileChannelIT.java similarity index 51% rename from community/io/src/test/java/org/neo4j/io/pagecache/impl/SingleFilePageSwapperWithAdversarialFileDispatcherIT.java rename to community/io/src/test/java/org/neo4j/io/pagecache/impl/SingleFilePageSwapperWithAdversarialFileChannelIT.java index 22ca24ad4d39c..17f2fb52e16ae 100644 --- a/community/io/src/test/java/org/neo4j/io/pagecache/impl/SingleFilePageSwapperWithAdversarialFileDispatcherIT.java +++ b/community/io/src/test/java/org/neo4j/io/pagecache/impl/SingleFilePageSwapperWithAdversarialFileChannelIT.java @@ -19,27 +19,14 @@ */ package org.neo4j.io.pagecache.impl; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.runner.RunWith; +import org.neo4j.adversaries.fs.AdversarialChannelDefaultFileSystemAbstraction; +import org.neo4j.io.fs.FileSystemAbstraction; -import org.neo4j.adversaries.fs.AdversarialFileChannel; -import org.neo4j.test.bootclasspathrunner.BootClassPathRunner; -import org.neo4j.unsafe.impl.internal.dragons.UnsafeUtilTest; - -@BootClassPathRunner.BootEntryOf( UnsafeUtilTest.class ) -@RunWith( BootClassPathRunner.class ) -public class SingleFilePageSwapperWithAdversarialFileDispatcherIT extends SingleFilePageSwapperWithRealFileSystemIT +public class SingleFilePageSwapperWithAdversarialFileChannelIT extends SingleFilePageSwapperWithRealFileSystemIT { - @BeforeClass - public static void enableAdversarialFileDispatcher() - { - AdversarialFileChannel.useAdversarialFileDispatcherHack = true; - } - - @AfterClass - public static void disableAdversarialFileDispatcher() + @Override + protected FileSystemAbstraction getFs() { - AdversarialFileChannel.useAdversarialFileDispatcherHack = false; + return new AdversarialChannelDefaultFileSystemAbstraction(); } } diff --git a/community/io/src/test/java/org/neo4j/io/pagecache/impl/muninn/MuninnPageCacheFixture.java b/community/io/src/test/java/org/neo4j/io/pagecache/impl/muninn/MuninnPageCacheFixture.java index 4b5168e089f62..6b098d6acd372 100644 --- a/community/io/src/test/java/org/neo4j/io/pagecache/impl/muninn/MuninnPageCacheFixture.java +++ b/community/io/src/test/java/org/neo4j/io/pagecache/impl/muninn/MuninnPageCacheFixture.java @@ -26,7 +26,7 @@ import org.neo4j.io.pagecache.PageSwapperFactory; import org.neo4j.io.pagecache.tracing.PageCacheTracer; -class MuninnPageCacheFixture extends PageCacheTestSupport.Fixture +public class MuninnPageCacheFixture extends PageCacheTestSupport.Fixture { CountDownLatch backgroundFlushLatch; diff --git a/community/io/src/test/java/org/neo4j/io/pagecache/impl/muninn/MuninnPageCacheSlowTestWithAdversarialFileDispatcherIT.java b/community/io/src/test/java/org/neo4j/io/pagecache/impl/muninn/MuninnPageCacheSlowTestWithAdversarialChannel.java similarity index 52% rename from community/io/src/test/java/org/neo4j/io/pagecache/impl/muninn/MuninnPageCacheSlowTestWithAdversarialFileDispatcherIT.java rename to community/io/src/test/java/org/neo4j/io/pagecache/impl/muninn/MuninnPageCacheSlowTestWithAdversarialChannel.java index 15839e78c3751..9e1e3dd01374a 100644 --- a/community/io/src/test/java/org/neo4j/io/pagecache/impl/muninn/MuninnPageCacheSlowTestWithAdversarialFileDispatcherIT.java +++ b/community/io/src/test/java/org/neo4j/io/pagecache/impl/muninn/MuninnPageCacheSlowTestWithAdversarialChannel.java @@ -19,27 +19,17 @@ */ package org.neo4j.io.pagecache.impl.muninn; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.runner.RunWith; +import org.neo4j.adversaries.fs.AdversarialChannelDefaultFileSystemAbstraction; -import org.neo4j.adversaries.fs.AdversarialFileChannel; -import org.neo4j.test.bootclasspathrunner.BootClassPathRunner; -import org.neo4j.unsafe.impl.internal.dragons.UnsafeUtilTest; - -@BootClassPathRunner.BootEntryOf( UnsafeUtilTest.class ) -@RunWith( BootClassPathRunner.class ) -public class MuninnPageCacheSlowTestWithAdversarialFileDispatcherIT extends MuninnPageCacheSlowTestWithRealFileSystemIT +public class MuninnPageCacheSlowTestWithAdversarialChannel extends MuninnPageCacheSlowTestWithRealFileSystemIT { - @BeforeClass - public static void enableAdversarialFileDispatcher() - { - AdversarialFileChannel.useAdversarialFileDispatcherHack = true; - } - @AfterClass - public static void disableAdversarialFileDispatcher() + @Override + protected Fixture createFixture() { - AdversarialFileChannel.useAdversarialFileDispatcherHack = false; + return super.createFixture() + .withFileSystemAbstraction( AdversarialChannelDefaultFileSystemAbstraction::new ) + .withFileConstructor( pathname -> directory.file( pathname ) ); } + } diff --git a/community/io/src/test/java/org/neo4j/io/pagecache/impl/muninn/MuninnPageCacheWithAdversarialFileDispatcherIT.java b/community/io/src/test/java/org/neo4j/io/pagecache/impl/muninn/MuninnPageCacheWithAdversarialFileDispatcherIT.java deleted file mode 100644 index 6ddf7243c5b60..0000000000000 --- a/community/io/src/test/java/org/neo4j/io/pagecache/impl/muninn/MuninnPageCacheWithAdversarialFileDispatcherIT.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright (c) 2002-2016 "Neo Technology," - * Network Engine for Objects in Lund AB [http://neotechnology.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package org.neo4j.io.pagecache.impl.muninn; - -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.runner.RunWith; - -import org.neo4j.adversaries.fs.AdversarialFileChannel; -import org.neo4j.test.bootclasspathrunner.BootClassPathRunner; -import org.neo4j.unsafe.impl.internal.dragons.UnsafeUtilTest; - -/** - * Use a special runner that will put the AccessibleFileDispatcher and the DelegateFileDispatcher in the -Xbootclasspath - * - * We already have the test-jar from the 'unsafe' module on our classpath. This custom test runner will put that - * classpath entry on the boot classpath. We refer to that classpath entry through the UnsafeUtilTest class, because - * we are not allowed to refer to the AccessibleFileDispatcher and the DelegateFileDispatcher classes directly. If we - * try to load them normally, we will be told that they are not allowed to refer to the sun.nio.ch.FileDispatcher - * superclass. - * - * To implement the boot classpath hack, the BootClassPathRunner will run the test in a sub-process, and tunnel the - * JUnit communication through RMI. - */ -@BootClassPathRunner.BootEntryOf( UnsafeUtilTest.class ) -@RunWith( BootClassPathRunner.class ) -public class MuninnPageCacheWithAdversarialFileDispatcherIT extends MuninnPageCacheWithRealFileSystemIT -{ - @BeforeClass - public static void enableAdversarialFileDispatcher() - { - AdversarialFileChannel.useAdversarialFileDispatcherHack = true; - } - - @AfterClass - public static void disableAdversarialFileDispatcher() - { - AdversarialFileChannel.useAdversarialFileDispatcherHack = false; - } -} diff --git a/community/io/src/test/java/org/neo4j/test/bootclasspathrunner/BootClassPathRunner.java b/community/io/src/test/java/org/neo4j/test/bootclasspathrunner/BootClassPathRunner.java deleted file mode 100644 index a794e0a96ed1b..0000000000000 --- a/community/io/src/test/java/org/neo4j/test/bootclasspathrunner/BootClassPathRunner.java +++ /dev/null @@ -1,147 +0,0 @@ -/* - * Copyright (c) 2002-2016 "Neo Technology," - * Network Engine for Objects in Lund AB [http://neotechnology.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package org.neo4j.test.bootclasspathrunner; - -import org.junit.runner.Description; -import org.junit.runner.Runner; -import org.junit.runner.notification.RunNotifier; -import org.junit.runners.BlockJUnit4ClassRunner; -import org.junit.runners.model.InitializationError; - -import java.io.File; -import java.lang.annotation.ElementType; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.annotation.Target; -import java.net.URL; -import java.rmi.registry.LocateRegistry; -import java.rmi.registry.Registry; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -import org.neo4j.io.proc.ProcessUtil; - -public class BootClassPathRunner extends Runner -{ - private static final String RMI_RUN_NOTIFIER_NAME = "RunNotifier"; - - @Target( ElementType.TYPE ) - @Retention( RetentionPolicy.RUNTIME ) - public @interface BootEntryOf - { - Class value(); - } - - private final Class testClass; - private final String classpathEntryToBootWith; - - @SuppressWarnings( "unchecked" ) - public BootClassPathRunner( Class testClass ) throws Exception - { - this.testClass = testClass; - BootEntryOf bootEntryOf = testClass.getAnnotation( BootEntryOf.class ); - Class bootEntryOfClass = bootEntryOf.value(); - URL location = bootEntryOfClass.getProtectionDomain().getCodeSource().getLocation(); - this.classpathEntryToBootWith = location.getPath(); - } - - @Override - public Description getDescription() - { - try - { - Runner describingRunner = new BlockJUnit4ClassRunner( testClass ); - return describingRunner.getDescription(); - } - catch ( InitializationError initializationError ) - { - throw new RuntimeException( initializationError ); - } - } - - @Override - public void run( RunNotifier notifier ) - { - StringBuilder classpath = buildClassPath(); - - try ( RmiServer server = new RmiServer() ) - { - int port = server.getPort(); - server.export( RMI_RUN_NOTIFIER_NAME, new DelegatingRemoteRunNotifier( notifier ) ); - - List command = new ArrayList<>(); - command.add( ProcessUtil.getJavaExecutable().toString() ); - for ( String argument : ProcessUtil.getJavaExecutableArguments() ) - { - if ( !argument.startsWith( "-agentlib" ) ) - { - command.add( argument ); - } - } - command.add( "-ea" ); - command.add( "-Xbootclasspath/a:" + classpathEntryToBootWith ); - command.add( "-cp" ); - command.add( classpath.toString() ); - command.add( getClass().getName() ); - command.add( String.valueOf( port ) ); - command.add( testClass.getName() ); - - ProcessBuilder pb = new ProcessBuilder(); - pb.command( command ); - pb.inheritIO(); - Process process = pb.start(); - process.waitFor(); - } - catch ( Exception e ) - { - throw new RuntimeException( e ); - } - } - - private StringBuilder buildClassPath() - { - Set classpathEntries = new HashSet<>(); - classpathEntries.addAll( ProcessUtil.getClassPathList() ); - classpathEntries.remove( classpathEntryToBootWith ); - StringBuilder classpath = new StringBuilder(); - for ( String classpathEntry : classpathEntries ) - { - classpath.append( classpathEntry ); - classpath.append( File.pathSeparator ); - } - classpath.setLength( classpath.length() - 1 ); // Cut off the last pathSeparator - return classpath; - } - - public static void main( String[] args ) throws Exception - { - int rmiPort = Integer.parseInt( args[0] ); - String testClassName = args[1]; - Class testClass = Class.forName( testClassName ); - - Registry registry = LocateRegistry.getRegistry( rmiPort ); - RemoteRunNotifier remote = (RemoteRunNotifier) registry.lookup( RMI_RUN_NOTIFIER_NAME ); - - Runner runner = new BlockJUnit4ClassRunner( testClass ); - runner.run( new DelegatingRunNotifier( remote ) ); - } -} diff --git a/community/io/src/test/java/org/neo4j/test/bootclasspathrunner/DelegatingRemoteRunNotifier.java b/community/io/src/test/java/org/neo4j/test/bootclasspathrunner/DelegatingRemoteRunNotifier.java deleted file mode 100644 index e8586738a3175..0000000000000 --- a/community/io/src/test/java/org/neo4j/test/bootclasspathrunner/DelegatingRemoteRunNotifier.java +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Copyright (c) 2002-2016 "Neo Technology," - * Network Engine for Objects in Lund AB [http://neotechnology.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package org.neo4j.test.bootclasspathrunner; - -import org.junit.runner.Description; -import org.junit.runner.Result; -import org.junit.runner.notification.Failure; -import org.junit.runner.notification.RunListener; -import org.junit.runner.notification.RunNotifier; -import org.junit.runner.notification.StoppedByUserException; - -class DelegatingRemoteRunNotifier implements RemoteRunNotifier -{ - private final RunNotifier delegate; - - public DelegatingRemoteRunNotifier( RunNotifier delegate ) - { - this.delegate = delegate; - } - - @Override - public void addListener( RunListener listener ) - { - delegate.addListener( listener ); - } - - @Override - public void fireTestRunFinished( Result result ) - { - delegate.fireTestRunFinished( result ); - } - - @Override - public void pleaseStop() - { - delegate.pleaseStop(); - } - - @Override - public void fireTestIgnored( Description description ) - { - delegate.fireTestIgnored( description ); - } - - @Override - public void fireTestStarted( Description description ) throws StoppedByUserException - { - delegate.fireTestStarted( description ); - } - - @Override - public void fireTestFinished( Description description ) - { - delegate.fireTestFinished( description ); - } - - @Override - public void fireTestAssumptionFailed( Failure failure ) - { - delegate.fireTestAssumptionFailed( failure ); - } - - @Override - public void addFirstListener( RunListener listener ) - { - delegate.addFirstListener( listener ); - } - - @Override - public void removeListener( RunListener listener ) - { - delegate.removeListener( listener ); - } - - @Override - public void fireTestRunStarted( Description description ) - { - delegate.fireTestRunStarted( description ); - } - - @Override - public void fireTestFailure( Failure failure ) - { - delegate.fireTestFailure( failure ); - } -} diff --git a/community/io/src/test/java/org/neo4j/test/bootclasspathrunner/DelegatingRunNotifier.java b/community/io/src/test/java/org/neo4j/test/bootclasspathrunner/DelegatingRunNotifier.java deleted file mode 100644 index 7e9919d83a170..0000000000000 --- a/community/io/src/test/java/org/neo4j/test/bootclasspathrunner/DelegatingRunNotifier.java +++ /dev/null @@ -1,171 +0,0 @@ -/* - * Copyright (c) 2002-2016 "Neo Technology," - * Network Engine for Objects in Lund AB [http://neotechnology.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package org.neo4j.test.bootclasspathrunner; - -import org.junit.runner.Description; -import org.junit.runner.Result; -import org.junit.runner.notification.Failure; -import org.junit.runner.notification.RunListener; -import org.junit.runner.notification.RunNotifier; -import org.junit.runner.notification.StoppedByUserException; - -import java.rmi.RemoteException; - -class DelegatingRunNotifier extends RunNotifier -{ - private final RemoteRunNotifier delegate; - - public DelegatingRunNotifier( RemoteRunNotifier delegate ) - { - this.delegate = delegate; - } - - public void addListener( RunListener listener ) - { - try - { - delegate.addListener( listener ); - } - catch ( RemoteException e ) - { - throw new RuntimeException( e ); - } - } - - public void fireTestFinished( Description description ) - { - try - { - delegate.fireTestFinished( description ); - } - catch ( RemoteException e ) - { - throw new RuntimeException( e ); - } - } - - public void fireTestIgnored( Description description ) - { - try - { - delegate.fireTestIgnored( description ); - } - catch ( RemoteException e ) - { - throw new RuntimeException( e ); - } - } - - public void fireTestAssumptionFailed( Failure failure ) - { - try - { - delegate.fireTestAssumptionFailed( failure ); - } - catch ( RemoteException e ) - { - throw new RuntimeException( e ); - } - } - - public void fireTestFailure( Failure failure ) - { - try - { - delegate.fireTestFailure( failure ); - } - catch ( RemoteException e ) - { - throw new RuntimeException( e ); - } - } - - public void addFirstListener( RunListener listener ) - { - try - { - delegate.addFirstListener( listener ); - } - catch ( RemoteException e ) - { - throw new RuntimeException( e ); - } - } - - public void fireTestRunFinished( Result result ) - { - try - { - delegate.fireTestRunFinished( result ); - } - catch ( RemoteException e ) - { - throw new RuntimeException( e ); - } - } - - public void fireTestRunStarted( Description description ) - { - try - { - delegate.fireTestRunStarted( description ); - } - catch ( RemoteException e ) - { - throw new RuntimeException( e ); - } - } - - public void fireTestStarted( Description description ) throws StoppedByUserException - { - try - { - delegate.fireTestStarted( description ); - } - catch ( RemoteException e ) - { - throw new RuntimeException( e ); - } - } - - public void removeListener( RunListener listener ) - { - try - { - delegate.removeListener( listener ); - } - catch ( RemoteException e ) - { - throw new RuntimeException( e ); - } - } - - public void pleaseStop() - { - try - { - delegate.pleaseStop(); - } - catch ( RemoteException e ) - { - throw new RuntimeException( e ); - } - } -} diff --git a/community/io/src/test/java/org/neo4j/test/bootclasspathrunner/RemoteRunNotifier.java b/community/io/src/test/java/org/neo4j/test/bootclasspathrunner/RemoteRunNotifier.java deleted file mode 100644 index 968768013abab..0000000000000 --- a/community/io/src/test/java/org/neo4j/test/bootclasspathrunner/RemoteRunNotifier.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright (c) 2002-2016 "Neo Technology," - * Network Engine for Objects in Lund AB [http://neotechnology.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package org.neo4j.test.bootclasspathrunner; - -import org.junit.runner.Description; -import org.junit.runner.Result; -import org.junit.runner.notification.Failure; -import org.junit.runner.notification.RunListener; -import org.junit.runner.notification.StoppedByUserException; - -import java.rmi.Remote; -import java.rmi.RemoteException; - -interface RemoteRunNotifier extends Remote -{ - void addListener( RunListener listener ) throws RemoteException; - - void fireTestRunFinished( Result result ) throws RemoteException; - - void pleaseStop() throws RemoteException; - - void fireTestIgnored( Description description ) throws RemoteException; - - void fireTestStarted( Description description ) throws StoppedByUserException, RemoteException; - - void fireTestFinished( Description description ) throws RemoteException; - - void fireTestAssumptionFailed( Failure failure ) throws RemoteException; - - void addFirstListener( RunListener listener ) throws RemoteException; - - void removeListener( RunListener listener ) throws RemoteException; - - void fireTestRunStarted( Description description ) throws RemoteException; - - void fireTestFailure( Failure failure ) throws RemoteException; -} diff --git a/community/io/src/test/java/org/neo4j/test/bootclasspathrunner/RmiServer.java b/community/io/src/test/java/org/neo4j/test/bootclasspathrunner/RmiServer.java deleted file mode 100644 index c6a9f29e26a5b..0000000000000 --- a/community/io/src/test/java/org/neo4j/test/bootclasspathrunner/RmiServer.java +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Copyright (c) 2002-2016 "Neo Technology," - * Network Engine for Objects in Lund AB [http://neotechnology.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package org.neo4j.test.bootclasspathrunner; - -import java.rmi.Remote; -import java.rmi.RemoteException; -import java.rmi.registry.LocateRegistry; -import java.rmi.registry.Registry; -import java.rmi.server.UnicastRemoteObject; -import java.util.ArrayList; -import java.util.List; - -class RmiServer implements AutoCloseable -{ - // The Registry is only initialised once per JVM, because otherwise RMI will complain - private static Registry registry; - private static int port = Registry.REGISTRY_PORT; - - private List boundNames = new ArrayList<>(); - - public RmiServer() - { - createRegistry(); - - } - - private static synchronized void createRegistry() - { - while ( registry == null ) - { - try - { - registry = LocateRegistry.createRegistry( port ); - } - catch ( RemoteException e ) - { - port++; - if ( port > Registry.REGISTRY_PORT + 4000 ) - { - throw new RuntimeException( e ); - } - } - } - } - - public int getPort() - { - return port; - } - - public void export( String name, Remote remote ) - { - try - { - Remote stub = UnicastRemoteObject.exportObject( remote, port ); - registry.rebind( name, stub ); - } - catch ( RemoteException e ) - { - throw new RuntimeException( e ); - } - } - - @Override - public void close() throws Exception - { - for ( String name : registry.list() ) - { - registry.unbind( name ); - } - } -} diff --git a/community/unsafe/src/test/java/sun/nio/ch/AccessibleFileDisptacher.java b/community/unsafe/src/test/java/sun/nio/ch/AccessibleFileDisptacher.java deleted file mode 100644 index c7d71aeb11f18..0000000000000 --- a/community/unsafe/src/test/java/sun/nio/ch/AccessibleFileDisptacher.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright (c) 2002-2016 "Neo Technology," - * Network Engine for Objects in Lund AB [http://neotechnology.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package sun.nio.ch; - -import java.io.FileDescriptor; -import java.io.IOException; - -public abstract class AccessibleFileDisptacher extends FileDispatcher -{ - // FileDispatcher: - public abstract int force(FileDescriptor fd, boolean metaData) throws IOException; - public abstract int force(FileDescriptor fd, boolean metaData, boolean writable) throws IOException; - public abstract int truncate(FileDescriptor fd, long size) throws IOException; - public abstract long size(FileDescriptor fd) throws IOException; - public abstract int lock(FileDescriptor fd, boolean blocking, long pos, long size, boolean shared) throws IOException; - public abstract void release(FileDescriptor fd, long pos, long size) throws IOException; - public abstract FileDescriptor duplicateForMapping(FileDescriptor fd) throws IOException; - - // NativeDispatcher: - public abstract int read(FileDescriptor fd, long address, int len) throws IOException; - public boolean needsPositionLock() { return super.needsPositionLock(); } - public int pread(FileDescriptor fd, long address, int len, long position) throws IOException { return super.pread( fd, address, len, position ); } - public abstract long readv(FileDescriptor fd, long address, int len) throws IOException; - public abstract int write(FileDescriptor fd, long address, int len) throws IOException; - public int pwrite(FileDescriptor fd, long address, int len, long position) throws IOException { return super.pwrite( fd, address, len, position ); } - public abstract long writev(FileDescriptor fd, long address, int len) throws IOException; - public abstract void close(FileDescriptor fd) throws IOException; - public void preClose(FileDescriptor fd) throws IOException { } -} diff --git a/community/unsafe/src/test/java/sun/nio/ch/DelegateFileDispatcher.java b/community/unsafe/src/test/java/sun/nio/ch/DelegateFileDispatcher.java deleted file mode 100644 index c1eb8f6bba19a..0000000000000 --- a/community/unsafe/src/test/java/sun/nio/ch/DelegateFileDispatcher.java +++ /dev/null @@ -1,228 +0,0 @@ -/* - * Copyright (c) 2002-2016 "Neo Technology," - * Network Engine for Objects in Lund AB [http://neotechnology.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package sun.nio.ch; - -import java.io.FileDescriptor; -import java.io.IOException; -import java.lang.invoke.MethodHandle; -import java.lang.invoke.MethodType; -import java.lang.reflect.Method; -import java.nio.channels.SelectableChannel; - -import org.neo4j.unsafe.impl.internal.dragons.LookupBypass; - -public class DelegateFileDispatcher extends AccessibleFileDisptacher -{ - private final FileDispatcher delegate; - - public DelegateFileDispatcher( Object delegate ) - { - this.delegate = (FileDispatcher) delegate; - } - - // Specific to all platforms other than Oracle JDK 7 on Linux, it seems - public int force( FileDescriptor fd, boolean metaData ) throws IOException - { - try - { - return (int) forceHandle.invokeExact( delegate, fd, metaData ); - } - catch ( Throwable throwable ) - { - throw new LinkageError( "not linked: FileDispatcher.force(FileDescriptor, boolean)", throwable ); - } - } - - // Specific to Oracle JDK 7 on Linux - public int force( FileDescriptor fd, boolean metaData, boolean writable ) throws IOException - { - try - { - return (int) forceHandle.invokeExact( delegate, fd, metaData, writable ); - } - catch ( Throwable throwable ) - { - throw new LinkageError( "not linked: FileDispatcher.force(FileDescriptor, boolean, boolean)", throwable ); - } - } - - private static final MethodHandle forceHandle = getForceHandle(); - - private static MethodHandle getForceHandle() - { - LookupBypass lookup = new LookupBypass(); - MethodType arg2 = MethodType.methodType( int.class, FileDescriptor.class, boolean.class ); - MethodType arg3 = MethodType.methodType( int.class, FileDescriptor.class, boolean.class, boolean.class ); - MethodHandle handle = lookupFileDispatcherHandle( lookup, "force", arg2, false ); - if ( handle == null ) - { - handle = lookupFileDispatcherHandle( lookup, "force", arg3, true ); - } - return handle; - } - - public long readv( FileDescriptor fd, long address, int len ) throws IOException - { - return delegate.readv( fd, address, len ); - } - - public int lock( FileDescriptor fd, boolean blocking, long pos, long size, boolean shared ) throws IOException - { - return delegate.lock( fd, blocking, pos, size, shared ); - } - - public FileDescriptor duplicateForMapping( FileDescriptor fd ) throws IOException - { - return delegate.duplicateForMapping( fd ); - } - - public int read( FileDescriptor fd, long address, int len ) throws IOException - { - return delegate.read( fd, address, len ); - } - - public int pwrite( FileDescriptor fd, long address, int len, long position ) throws IOException - { - return delegate.pwrite( fd, address, len, position ); - } - - public void preClose( FileDescriptor fd ) throws IOException - { - delegate.preClose( fd ); - } - - public int truncate( FileDescriptor fd, long size ) throws IOException - { - return delegate.truncate( fd, size ); - } - - public void release( FileDescriptor fd, long pos, long size ) throws IOException - { - delegate.release( fd, pos, size ); - } - - public boolean needsPositionLock() - { - return delegate.needsPositionLock(); - } - - public long size( FileDescriptor fd ) throws IOException - { - return delegate.size( fd ); - } - - public int pread( FileDescriptor fd, long address, int len, long position ) throws IOException - { - return delegate.pread( fd, address, len, position ); - } - - public long writev( FileDescriptor fd, long address, int len ) throws IOException - { - return delegate.writev( fd, address, len ); - } - - public int write( FileDescriptor fd, long address, int len ) throws IOException - { - return delegate.write( fd, address, len ); - } - - public void close( FileDescriptor fd ) throws IOException - { - delegate.close( fd ); - } - - // new method introduced in oracle 8u60 and 7u80 - boolean transferToDirectlyNeedsPositionLock() - { - if ( transferToDirectlyNeedsPositionLockHandle == null ) - { - // whatever it should not be called... - return false; - } - - try - { - return (Boolean) transferToDirectlyNeedsPositionLockHandle.invokeExact( delegate ); - } - catch ( Throwable throwable ) - { - throw new LinkageError( "not linked: FileDispatcher.transferToDirectlyNeedsPositionLock()", throwable ); - } - } - - private static final MethodHandle transferToDirectlyNeedsPositionLockHandle = - getTransferToDirectlyNeedsPositionLockHandle(); - - public static MethodHandle getTransferToDirectlyNeedsPositionLockHandle() - { - LookupBypass lookup = new LookupBypass(); - MethodType methodType = MethodType.methodType( boolean.class ); - return lookupFileDispatcherHandle( lookup, "transferToDirectlyNeedsPositionLock", methodType, false ); - } - - // new method introduced in oracle 8u60 and 7u80 - boolean canTransferToDirectly( SelectableChannel selectableChannel ) - { - if ( canTransferToDirectlyHandle == null ) - { - // whatever it should not be called... - return false; - } - - try - { - return (Boolean) canTransferToDirectlyHandle.invokeExact( delegate, selectableChannel ); - } - catch ( Throwable throwable ) - { - throw new LinkageError( "not linked: FileDispatcher.canTransferToDirectly(SelectableChannel)", throwable ); - } - } - - private static final MethodHandle canTransferToDirectlyHandle = getCanTransferToDirectlyHandle(); - - public static MethodHandle getCanTransferToDirectlyHandle() - { - LookupBypass lookup = new LookupBypass(); - MethodType methodType = MethodType.methodType( boolean.class, SelectableChannel.class ); - return lookupFileDispatcherHandle( lookup, "canTransferToDirectly", methodType, false ); - } - - // utility method - private static MethodHandle lookupFileDispatcherHandle( - LookupBypass lookup, String name, MethodType type, boolean throwOnError ) - { - try - { - Class cls = FileDispatcher.class; - Method method = cls.getDeclaredMethod( name, type.parameterArray() ); - method.setAccessible( true ); - return lookup.unreflect( method ); // We have to unreflect because we need to setAccessible( true ) - } - catch ( Exception e ) - { - if ( throwOnError ) - { - throw new LinkageError( "No such FileDispatcher method: " + name + " of type " + type ); - } - return null; - } - } -}