From 0275570e5077e61ec09fea684a4397732a52181f Mon Sep 17 00:00:00 2001 From: lutovich Date: Thu, 29 Oct 2015 11:47:27 +0100 Subject: [PATCH] Rewritten/removed tests that use JDI and breakpoints * removed TestConcurrentRotation. it was supposed to verify that it is possible to load legacy index while rotating lucene transaction log. Currently we only have a single transaction log and rotation happens in a dedicated checkpointer thread concurrently with transactions. That is why this test does not seem to be relevant any more * TestPropertyReadOnNewEntityBeforeLockRelease rewritten in stochastic way to not use JDI and breakpoints * Removed breakpoints from IdGeneratorRebuildFailureEmulationTest. Breakpoint enabling annotations were just leftovers and not actually used. --- ...dGeneratorRebuildFailureEmulationTest.java | 37 +-- .../TestConcurrentRotation.java | 210 ---------------- ...pertyReadOnNewEntityBeforeLockRelease.java | 236 +++++++++--------- 3 files changed, 127 insertions(+), 356 deletions(-) delete mode 100644 community/neo4j/src/test/java/synchronization/TestConcurrentRotation.java diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/IdGeneratorRebuildFailureEmulationTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/IdGeneratorRebuildFailureEmulationTest.java index b38e402ad1a04..22c4037827fa2 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/IdGeneratorRebuildFailureEmulationTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/IdGeneratorRebuildFailureEmulationTest.java @@ -51,8 +51,6 @@ import org.neo4j.logging.NullLogProvider; import org.neo4j.test.ImpermanentGraphDatabase; import org.neo4j.test.PageCacheRule; -import org.neo4j.test.subprocess.BreakpointTrigger; -import org.neo4j.test.subprocess.EnabledBreakpoints; import org.neo4j.tooling.GlobalGraphOperations; import static org.hamcrest.Matchers.startsWith; @@ -75,10 +73,9 @@ protected void emulateFailureOnRebuildOf( NeoStores neoStores ) } } - @BreakpointTrigger - private void performTest() throws Exception + private void performTest( String neostoreFileName ) throws Exception { - File idFile = new File( storeDir, Thread.currentThread().getStackTrace()[2].getMethodName().replace( '_', '.' ) + ".id" ); + File idFile = new File( storeDir, neostoreFileName + ".id" ); // emulate the need for rebuilding id generators by deleting it fs.deleteFile( idFile ); NeoStores neoStores = null; @@ -256,74 +253,64 @@ protected FileSystemAbstraction createFileSystemAbstraction() } - @EnabledBreakpoints("performTest") @Test public void neostore() throws Exception { - performTest(); + performTest( MetaDataStore.DEFAULT_NAME ); } - @EnabledBreakpoints("performTest") @Test public void neostore_nodestore_db() throws Exception { - performTest(); + performTest( MetaDataStore.DEFAULT_NAME + StoreFactory.NODE_STORE_NAME ); } - @EnabledBreakpoints("performTest") @Test public void neostore_propertystore_db_arrays() throws Exception { - performTest(); + performTest( MetaDataStore.DEFAULT_NAME + StoreFactory.PROPERTY_ARRAYS_STORE_NAME ); } - @EnabledBreakpoints("performTest") @Test public void neostore_propertystore_db() throws Exception { - performTest(); + performTest( MetaDataStore.DEFAULT_NAME + StoreFactory.PROPERTY_STORE_NAME ); } - @EnabledBreakpoints("performTest") @Test public void neostore_propertystore_db_index() throws Exception { - performTest(); + performTest( MetaDataStore.DEFAULT_NAME + StoreFactory.PROPERTY_KEY_TOKEN_STORE_NAME ); } - @EnabledBreakpoints("performTest") @Test public void neostore_propertystore_db_index_keys() throws Exception { - performTest(); + performTest( MetaDataStore.DEFAULT_NAME + StoreFactory.PROPERTY_KEY_TOKEN_NAMES_STORE_NAME ); } - @EnabledBreakpoints("performTest") @Test public void neostore_propertystore_db_strings() throws Exception { - performTest(); + performTest( MetaDataStore.DEFAULT_NAME + StoreFactory.PROPERTY_STRINGS_STORE_NAME ); } - @EnabledBreakpoints("performTest") @Test public void neostore_relationshipstore_db() throws Exception { - performTest(); + performTest( MetaDataStore.DEFAULT_NAME + StoreFactory.RELATIONSHIP_STORE_NAME ); } - @EnabledBreakpoints("performTest") @Test public void neostore_relationshiptypestore_db() throws Exception { - performTest(); + performTest( MetaDataStore.DEFAULT_NAME + StoreFactory.RELATIONSHIP_TYPE_TOKEN_STORE_NAME ); } - @EnabledBreakpoints("performTest") @Test public void neostore_relationshiptypestore_db_names() throws Exception { - performTest(); + performTest( MetaDataStore.DEFAULT_NAME + StoreFactory.RELATIONSHIP_TYPE_TOKEN_NAMES_STORE_NAME ); } private IdGeneratorRebuildFailureEmulationTest() diff --git a/community/neo4j/src/test/java/synchronization/TestConcurrentRotation.java b/community/neo4j/src/test/java/synchronization/TestConcurrentRotation.java deleted file mode 100644 index f92c0acc1f650..0000000000000 --- a/community/neo4j/src/test/java/synchronization/TestConcurrentRotation.java +++ /dev/null @@ -1,210 +0,0 @@ -/* - * Copyright (c) 2002-2015 "Neo Technology," - * Network Engine for Objects in Lund AB [http://neotechnology.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package synchronization; - -import org.apache.lucene.index.IndexWriter; -import org.junit.Test; - -import java.io.IOException; -import java.util.concurrent.CountDownLatch; - -import org.neo4j.graphdb.Node; -import org.neo4j.graphdb.Transaction; -import org.neo4j.helpers.Exceptions; -import org.neo4j.kernel.GraphDatabaseAPI; -import org.neo4j.kernel.impl.transaction.log.checkpoint.CheckPointer; -import org.neo4j.kernel.impl.transaction.log.checkpoint.SimpleTriggerInfo; -import org.neo4j.test.AbstractSubProcessTestBase; -import org.neo4j.test.subprocess.BreakPoint; -import org.neo4j.test.subprocess.DebugInterface; -import org.neo4j.test.subprocess.DebuggedThread; -import org.neo4j.test.subprocess.KillSubProcess; - -import static org.junit.Assert.assertTrue; - -public class TestConcurrentRotation extends AbstractSubProcessTestBase -{ - private final CountDownLatch barrier1 = new CountDownLatch( 1 ), barrier2 = new CountDownLatch( 1 ); - - private DebuggedThread thread; - - private final BreakPoint commitIndexWriter = new BreakPoint( IndexWriter.class, "commit" ) - { - private int counter = 0; - - @Override - protected void callback( DebugInterface debug ) throws KillSubProcess - { - if ( counter++ > 0 ) - { - return; - } - thread = debug.thread().suspend( this ); - this.disable(); - barrier1.countDown(); - } - }; - private final BreakPoint resumeFlushThread = new BreakPoint( TestConcurrentRotation.class, "resumeFlushThread" ) - { - @Override - protected void callback( DebugInterface debug ) throws KillSubProcess - { - thread.resume(); - this.disable(); - } - }; - - private final BreakPoint done = new BreakPoint( TestConcurrentRotation.class, "checkPointDone" ) - { - @Override - protected void callback( DebugInterface debug ) throws KillSubProcess - { - this.disable(); - barrier2.countDown(); - } - }; - - static void resumeFlushThread() - { // Activates breakpoint - } - - static void checkPointDone() - { // Activate breakpoint - } - - @Override - protected BreakPoint[] breakpoints( int id ) - { - return new BreakPoint[] { commitIndexWriter, resumeFlushThread.enable(), done.enable() }; - } - - @Test - public void rotateLogAtTheSameTimeInitializeIndexWriters() throws Exception - { - run( new CreateInitialStateTask() ); - restart(); - commitIndexWriter.enable(); - run( new LoadIndexesTask( 2, false ) ); - RotateIndexLogTask rotateTask = new RotateIndexLogTask(); - runInThread( rotateTask ); - barrier1.await(); - run( new LoadIndexesTask( 3, true ) ); - resumeFlushThread(); - barrier2.await(); - run( new Verifier() ); - } - - private static class Verifier implements Task - { - @Override - public void run( GraphDatabaseAPI graphdb ) - { - try(Transaction ignored = graphdb.beginTx()) - { - // TODO: Pass a node reference around of assuming the id will be deterministically assigned, - // artifact of removing the reference node, upon which this test used to depend. - assertTrue( (Boolean) graphdb.getNodeById(3).getProperty( "success" ) ); - } - } - } - - private static class CreateInitialStateTask implements Task - { - @Override - public void run( GraphDatabaseAPI graphdb ) - { - try(Transaction tx = graphdb.beginTx()) - { - for ( int i = 0; i < 3; i++ ) - { - graphdb.index().forNodes( "index" + i ).add( graphdb.createNode(), "name", "" + i ); - } - tx.success(); - } - } - } - - private static class LoadIndexesTask implements Task - { - private final int count; - private final boolean resume; - - public LoadIndexesTask( int count, boolean resume ) - { - this.count = count; - this.resume = resume; - } - - @Override - public void run( GraphDatabaseAPI graphdb ) - { - try(Transaction ignored = graphdb.beginTx()) - { - for ( int i = 0; i < count; i++ ) - { - graphdb.index().forNodes( "index" + i ).get( "name", i ).getSingle(); - } - } - if ( resume ) - { - resumeFlushThread(); - } - } - } - - private static class RotateIndexLogTask implements Task - { - @Override - public void run( GraphDatabaseAPI graphdb ) - { - try - { - checkPoint( graphdb ); - setSuccess( graphdb, true ); - } - catch ( Exception e ) - { - setSuccess( graphdb, false ); - throw Exceptions.launderedException( e ); - } - finally - { - checkPointDone(); - } - } - - private void checkPoint( GraphDatabaseAPI graphdb ) throws IOException - { - graphdb.getDependencyResolver().resolveDependency( CheckPointer.class ).forceCheckPoint( - new SimpleTriggerInfo( "test" ) - ); - } - - private void setSuccess( GraphDatabaseAPI graphdb, boolean success ) - { - try(Transaction tx = graphdb.beginTx()) - { - Node node = graphdb.createNode(); - node.setProperty( "success", success ); - tx.success(); - } - } - } -} diff --git a/community/neo4j/src/test/java/visibility/TestPropertyReadOnNewEntityBeforeLockRelease.java b/community/neo4j/src/test/java/visibility/TestPropertyReadOnNewEntityBeforeLockRelease.java index d6b8a1e20728e..29d7bf9140a97 100644 --- a/community/neo4j/src/test/java/visibility/TestPropertyReadOnNewEntityBeforeLockRelease.java +++ b/community/neo4j/src/test/java/visibility/TestPropertyReadOnNewEntityBeforeLockRelease.java @@ -19,169 +19,163 @@ */ package visibility; -import java.util.concurrent.CountDownLatch; - +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Rule; import org.junit.Test; +import java.util.UUID; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.neo4j.graphdb.GraphDatabaseService; import org.neo4j.graphdb.Node; import org.neo4j.graphdb.Transaction; -import org.neo4j.graphdb.factory.GraphDatabaseFactory; -import org.neo4j.kernel.GraphDatabaseAPI; -import org.neo4j.kernel.impl.api.KernelTransactionImplementation; -import org.neo4j.test.AbstractSubProcessTestBase; -import org.neo4j.test.subprocess.BreakPoint; -import org.neo4j.test.subprocess.DebugInterface; -import org.neo4j.test.subprocess.DebuggedThread; -import org.neo4j.test.subprocess.KillSubProcess; +import org.neo4j.test.DatabaseRule; +import org.neo4j.test.ImpermanentDatabaseRule; +import org.neo4j.test.RepeatRule; +import org.neo4j.test.RepeatRule.Repeat; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; -@SuppressWarnings( "serial" ) -public class TestPropertyReadOnNewEntityBeforeLockRelease extends AbstractSubProcessTestBase +public class TestPropertyReadOnNewEntityBeforeLockRelease { - private final CountDownLatch latch1 = new CountDownLatch( 1 ), latch2 = new CountDownLatch( 1 ); + private static final String INDEX_NAME = "nodes"; + private static final int MAX_READER_DELAY_MS = 10; - @Test - public void shouldBeAbleToReadPropertiesFromNewNodeReturnedFromIndex() throws Exception - { - runInThread( new CreateData() ); - latch1.await(); - run( new ReadData() ); - latch2.await(); - } + @ClassRule + public static final DatabaseRule db = new ImpermanentDatabaseRule(); + @Rule + public final RepeatRule repeat = new RepeatRule(); - private static class CreateData implements Task + @BeforeClass + public static void initializeIndex() throws Exception { - @Override - public void run( GraphDatabaseAPI graphdb ) + try ( Transaction tx = db.beginTx() ) { - try(Transaction tx = graphdb.beginTx()) - { - Node node = graphdb.createNode(); - node.setProperty( "value", "present" ); - graphdb.index().forNodes( "nodes" ).add( node, "value", "present" ); - enableBreakPoints(); - - tx.success(); - } - done(); + Node node = db.createNode(); + db.index().forNodes( INDEX_NAME ).add( node, "foo", "bar" ); + tx.success(); } } - static void enableBreakPoints() - { - // triggers breakpoint - } - - static void done() + @Test + @Repeat( times = 100 ) + public void shouldBeAbleToReadPropertiesFromNewNodeReturnedFromIndex() throws Exception { - // triggers breakpoint - } + String propertyKey = UUID.randomUUID().toString(); + String propertyValue = UUID.randomUUID().toString(); + AtomicBoolean start = new AtomicBoolean( false ); + int readerDelay = ThreadLocalRandom.current().nextInt( MAX_READER_DELAY_MS ); - static void resumeThread() - { - // triggers breakpoint - } + Writer writer = new Writer( db, propertyKey, propertyValue, start ); + Reader reader = new Reader( db, propertyKey, propertyValue, start, readerDelay ); - private static class ReadData implements Task - { - @Override - public void run( GraphDatabaseAPI graphdb ) + ExecutorService executor = Executors.newFixedThreadPool( 2 ); + Future readResult; + Future writeResult; + try { - try(Transaction ignored = graphdb.beginTx()) - { - Node node = graphdb.index().forNodes( "nodes" ).get( "value", "present" ).getSingle(); - assertNotNull( "did not get the node from the index", node ); - assertEquals( "present", node.getProperty( "value" ) ); - } - resumeThread(); - } - } + writeResult = executor.submit( writer ); + readResult = executor.submit( reader ); - private volatile DebuggedThread thread; - private final BreakPoint lockReleaserCommit = new BreakPoint( KernelTransactionImplementation.class, "release" ) - { - @Override - protected void callback( DebugInterface debug ) throws KillSubProcess - { - thread = debug.thread().suspend( this ); - resumeThread.enable(); - this.disable(); - latch1.countDown(); + start.set( true ); } - }, enableBreakPoints = new BreakPoint( TestPropertyReadOnNewEntityBeforeLockRelease.class, "enableBreakPoints" ) - { - @Override - protected void callback( DebugInterface debug ) throws KillSubProcess + finally { - lockReleaserCommit.enable(); - this.disable(); + executor.shutdown(); + executor.awaitTermination( 20, TimeUnit.SECONDS ); } - }, resumeThread = new BreakPoint( TestPropertyReadOnNewEntityBeforeLockRelease.class, "resumeThread" ) + + assertNull( writeResult.get() ); + assertNull( readResult.get() ); + } + + private static class Writer implements Runnable { - @Override - protected void callback( DebugInterface debug ) throws KillSubProcess + final GraphDatabaseService db; + final String propertyKey; + final String propertyValue; + final AtomicBoolean start; + + Writer( GraphDatabaseService db, String propertyKey, String propertyValue, AtomicBoolean start ) { - thread.resume(); - this.disable(); + this.db = db; + this.propertyKey = propertyKey; + this.propertyValue = propertyValue; + this.start = start; } - }, done = new BreakPoint( TestPropertyReadOnNewEntityBeforeLockRelease.class, "done" ) - { + @Override - protected void callback( DebugInterface debug ) throws KillSubProcess + public void run() { - latch2.countDown(); - this.disable(); + while ( !start.get() ) + { + // spin + } + try ( Transaction tx = db.beginTx() ) + { + Node node = db.createNode(); + node.setProperty( propertyKey, propertyValue ); + db.index().forNodes( INDEX_NAME ).add( node, propertyKey, propertyValue ); + tx.success(); + } } - }; - - @Override - protected BreakPoint[] breakpoints( int id ) - { - return new BreakPoint[] { lockReleaserCommit, enableBreakPoints.enable(), resumeThread, done.enable() }; } - /** - * Version of the test case useful for manual debugging. - */ - public static void main( String... args ) throws Exception + private static class Reader implements Runnable { - final GraphDatabaseAPI graphdb = (GraphDatabaseAPI) new GraphDatabaseFactory(). - newEmbeddedDatabase( "target/test-data/" + TestPropertyReadOnNewEntityBeforeLockRelease.class - .getName() + "/graphdb" ); - final CountDownLatch completion = new CountDownLatch( 2 ); - class TaskRunner implements Runnable + final GraphDatabaseService db; + final String propertyKey; + final String propertyValue; + final AtomicBoolean start; + private final int delay; + + Reader( GraphDatabaseService db, String propertyKey, String propertyValue, AtomicBoolean start, int delay ) { - private final Task task; + this.db = db; + this.propertyKey = propertyKey; + this.propertyValue = propertyValue; + this.start = start; + this.delay = delay; + } - TaskRunner( Task task ) + @Override + public void run() + { + while ( !start.get() ) { - this.task = task; + // spin } - - @Override - public void run() + sleep(); + try ( Transaction tx = db.beginTx() ) { - try - { - task.run( graphdb ); - } - finally + // it is acceptable to either see a node with correct property or not see it at all + Node node = db.index().forNodes( INDEX_NAME ).get( propertyKey, propertyValue ).getSingle(); + if ( node != null ) { - completion.countDown(); + assertEquals( propertyValue, node.getProperty( propertyKey ) ); } + tx.success(); } } - new Thread( new TaskRunner( new CreateData() ) ).start(); - new Thread( new TaskRunner( new ReadData() ) ).start(); - try - { - completion.await(); - } - finally + + private void sleep() { - graphdb.shutdown(); + try + { + Thread.sleep( delay ); + } + catch ( InterruptedException e ) + { + Thread.currentThread().interrupt(); + throw new RuntimeException( e ); + } } } }