Skip to content

Commit

Permalink
A bunch of tests avoids accessing NeoStores directly
Browse files Browse the repository at this point in the history 
instead goes via Kernel/Storage API
  • Loading branch information
@tinwelint
tinwelint committed Jun 19, 2018
1 parent 9a9c527 commit ae4c6d4
Show file tree
Hide file tree
Showing 11 changed files with 144 additions and 192 deletions.
60 changes: 21 additions & 39 deletions ...ity/community-it/index-it/src/test/java/org/neo4j/index/IndexSamplingIntegrationTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,21 +31,14 @@
import org.neo4j.graphdb.ResourceIterator;
import org.neo4j.graphdb.Transaction;
import org.neo4j.graphdb.schema.IndexDefinition;
import org.neo4j.internal.kernel.api.TokenRead;
import org.neo4j.internal.kernel.api.IndexReference;
import org.neo4j.internal.kernel.api.Kernel;
import org.neo4j.internal.kernel.api.exceptions.TransactionFailureException;
import org.neo4j.io.fs.FileUtils;
import org.neo4j.kernel.api.KernelTransaction;
import org.neo4j.kernel.api.Statement;
import org.neo4j.kernel.api.exceptions.index.IndexNotFoundKernelException;
import org.neo4j.kernel.api.schema.SchemaDescriptorFactory;
import org.neo4j.kernel.impl.api.index.IndexingService;
import org.neo4j.kernel.impl.core.ThreadToStatementContextBridge;
import org.neo4j.kernel.impl.storageengine.impl.recordstorage.RecordStorageEngine;
import org.neo4j.kernel.impl.store.MetaDataStore;
import org.neo4j.kernel.impl.store.StoreFactory;
import org.neo4j.kernel.impl.store.counts.CountsTracker;
import org.neo4j.kernel.impl.store.counts.keys.CountsKeyFactory;
import org.neo4j.kernel.impl.store.counts.keys.IndexSampleKey;
import org.neo4j.kernel.impl.store.counts.keys.IndexStatisticsKey;
import org.neo4j.kernel.internal.GraphDatabaseAPI;
import org.neo4j.register.Register.DoubleLongRegister;
import org.neo4j.register.Registers;
Expand All @@ -57,6 +50,8 @@
import static org.hamcrest.Matchers.lessThanOrEqualTo;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertThat;
import static org.neo4j.internal.kernel.api.Transaction.Type.explicit;
import static org.neo4j.internal.kernel.api.security.LoginContext.AUTH_DISABLED;

public class IndexSamplingIntegrationTest
{
Expand Down Expand Up @@ -195,40 +190,26 @@ public void shouldSampleUniqueIndex() throws Throwable
assertEquals( nodes - deletedNodes, indexSizeRegister.readSecond() );
}

private long indexId( GraphDatabaseAPI api ) throws IndexNotFoundKernelException
private IndexReference indexId( org.neo4j.internal.kernel.api.Transaction tx )
{
ThreadToStatementContextBridge contextBridge =
api.getDependencyResolver().resolveDependency( ThreadToStatementContextBridge.class );
try ( Transaction tx = api.beginTx() )
{
KernelTransaction ktx =
contextBridge.getKernelTransactionBoundToThisThread( true );
try ( Statement ignore = ktx.acquireStatement() )
{
IndexingService indexingService =
api.getDependencyResolver().resolveDependency( IndexingService.class );
TokenRead tokenRead = ktx.tokenRead();
int labelId = tokenRead.nodeLabel( label.name() );
int propertyKeyId = tokenRead.propertyKey( property );
long indexId = indexingService.getIndexId( SchemaDescriptorFactory.forLabel( labelId, propertyKeyId ) );
tx.success();
return indexId;
}
}
int labelId = tx.tokenRead().nodeLabel( label.name() );
int propertyKeyId = tx.tokenRead().propertyKey( property );
return tx.schemaRead().index( labelId, propertyKeyId );
}

private DoubleLongRegister fetchIndexSamplingValues( GraphDatabaseService db ) throws IndexNotFoundKernelException
private DoubleLongRegister fetchIndexSamplingValues( GraphDatabaseService db ) throws IndexNotFoundKernelException, TransactionFailureException
{
try
{
// Then
db = new TestGraphDatabaseFactory().newEmbeddedDatabase( testDirectory.graphDbDir() );
@SuppressWarnings( "deprecation" )
GraphDatabaseAPI api = (GraphDatabaseAPI) db;
CountsTracker countsTracker = api.getDependencyResolver().resolveDependency( RecordStorageEngine.class )
.testAccessNeoStores().getCounts();
IndexSampleKey key = CountsKeyFactory.indexSampleKey( indexId( api ) );
return countsTracker.get( key, Registers.newDoubleLongRegister() );
try ( org.neo4j.internal.kernel.api.Transaction tx = api.getDependencyResolver().resolveDependency( Kernel.class )
.beginTransaction( explicit, AUTH_DISABLED ) )
{
return tx.schemaRead().indexSample( indexId( tx ), Registers.newDoubleLongRegister() );
}
}
finally
{
Expand All @@ -239,18 +220,19 @@ private DoubleLongRegister fetchIndexSamplingValues( GraphDatabaseService db ) t
}
}

private DoubleLongRegister fetchIndexSizeValues( GraphDatabaseService db ) throws IndexNotFoundKernelException
private DoubleLongRegister fetchIndexSizeValues( GraphDatabaseService db ) throws IndexNotFoundKernelException, TransactionFailureException
{
try
{
// Then
db = new TestGraphDatabaseFactory().newEmbeddedDatabase( testDirectory.graphDbDir() );
@SuppressWarnings( "deprecation" )
GraphDatabaseAPI api = (GraphDatabaseAPI) db;
CountsTracker countsTracker = api.getDependencyResolver().resolveDependency( RecordStorageEngine.class )
.testAccessNeoStores().getCounts();
IndexStatisticsKey key = CountsKeyFactory.indexStatisticsKey( indexId( api ) );
return countsTracker.get( key, Registers.newDoubleLongRegister() );
try ( org.neo4j.internal.kernel.api.Transaction tx = api.getDependencyResolver().resolveDependency( Kernel.class )
.beginTransaction( explicit, AUTH_DISABLED ) )
{
return tx.schemaRead().indexUpdatesAndSize( indexId( tx ), Registers.newDoubleLongRegister() );
}
}
finally
{
Expand Down
39 changes: 21 additions & 18 deletions community/community-it/kernel-it/src/test/java/counts/RebuildCountsTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,11 +33,12 @@
import org.neo4j.graphdb.ResourceIterator;
import org.neo4j.graphdb.Transaction;
import org.neo4j.graphdb.mockfs.UncloseableDelegatingFileSystemAbstraction;
import org.neo4j.internal.kernel.api.Kernel;
import org.neo4j.internal.kernel.api.exceptions.TransactionFailureException;
import org.neo4j.io.fs.FileSystemAbstraction;
import org.neo4j.kernel.impl.api.index.inmemory.InMemoryIndexProvider;
import org.neo4j.kernel.impl.api.index.inmemory.InMemoryIndexProviderFactory;
import org.neo4j.kernel.impl.core.ThreadToStatementContextBridge;
import org.neo4j.kernel.impl.storageengine.impl.recordstorage.RecordStorageEngine;
import org.neo4j.kernel.impl.store.MetaDataStore;
import org.neo4j.kernel.impl.store.StoreFactory;
import org.neo4j.kernel.impl.store.counts.CountsTracker;
Expand All @@ -53,14 +54,16 @@
import static org.junit.Assert.assertTrue;
import static org.neo4j.graphdb.Label.label;
import static org.neo4j.graphdb.factory.GraphDatabaseSettings.index_background_sampling_enabled;

import static org.neo4j.logging.AssertableLogProvider.LogMatcherBuilder;
import static org.neo4j.internal.kernel.api.Transaction.Type.explicit;
import static org.neo4j.internal.kernel.api.security.LoginContext.AUTH_DISABLED;
import static org.neo4j.logging.AssertableLogProvider.inLog;
import static org.neo4j.register.Registers.newDoubleLongRegister;

public class RebuildCountsTest
{
@Test
public void shouldRebuildMissingCountsStoreOnStart() throws IOException
public void shouldRebuildMissingCountsStoreOnStart() throws IOException, TransactionFailureException
{
// given
createAliensAndHumans();
Expand All @@ -71,10 +74,13 @@ public void shouldRebuildMissingCountsStoreOnStart() throws IOException
restart( fs );

// then
CountsTracker tracker = counts();
assertEquals( ALIENS + HUMANS, tracker.nodeCount( -1, newDoubleLongRegister() ).readSecond() );
assertEquals( ALIENS, tracker.nodeCount( labelId( ALIEN ), newDoubleLongRegister() ).readSecond() );
assertEquals( HUMANS, tracker.nodeCount( labelId( HUMAN ), newDoubleLongRegister() ).readSecond() );
try ( org.neo4j.internal.kernel.api.Transaction tx = ((GraphDatabaseAPI)db).getDependencyResolver().resolveDependency( Kernel.class )
.beginTransaction( explicit, AUTH_DISABLED ) )
{
assertEquals( ALIENS + HUMANS, tx.dataRead().countsForNode( -1 ) );
assertEquals( ALIENS, tx.dataRead().countsForNode( labelId( ALIEN ) ) );
assertEquals( HUMANS, tx.dataRead().countsForNode( labelId( HUMAN ) ) );
}

// and also
LogMatcherBuilder matcherBuilder = inLog( MetaDataStore.class );
Expand All @@ -83,7 +89,7 @@ public void shouldRebuildMissingCountsStoreOnStart() throws IOException
}

@Test
public void shouldRebuildMissingCountsStoreAfterRecovery() throws IOException
public void shouldRebuildMissingCountsStoreAfterRecovery() throws IOException, TransactionFailureException
{
// given
createAliensAndHumans();
Expand All @@ -96,10 +102,13 @@ public void shouldRebuildMissingCountsStoreAfterRecovery() throws IOException
restart( fs );

// then
CountsTracker tracker = counts();
assertEquals( ALIENS, tracker.nodeCount( -1, newDoubleLongRegister() ).readSecond() );
assertEquals( ALIENS, tracker.nodeCount( labelId( ALIEN ), newDoubleLongRegister() ).readSecond() );
assertEquals( 0, tracker.nodeCount( labelId( HUMAN ), newDoubleLongRegister() ).readSecond() );
try ( org.neo4j.internal.kernel.api.Transaction tx = ((GraphDatabaseAPI)db).getDependencyResolver().resolveDependency( Kernel.class )
.beginTransaction( explicit, AUTH_DISABLED ) )
{
assertEquals( ALIENS, tx.dataRead().countsForNode( -1 ) );
assertEquals( ALIENS, tx.dataRead().countsForNode( labelId( ALIEN ) ) );
assertEquals( 0, tx.dataRead().countsForNode( labelId( HUMAN ) ) );
}

// and also
LogMatcherBuilder matcherBuilder = inLog( MetaDataStore.class );
Expand Down Expand Up @@ -149,12 +158,6 @@ private int labelId( Label alien )
}
}

private CountsTracker counts()
{
return ((GraphDatabaseAPI) db).getDependencyResolver().resolveDependency( RecordStorageEngine.class )
.testAccessNeoStores().getCounts();
}

private void deleteCounts( FileSystemAbstraction snapshot )
{
final File storeFileBase = new File( storeDir, MetaDataStore.DEFAULT_NAME + StoreFactory.COUNTS_STORE );
Expand Down
17 changes: 9 additions & 8 deletions community/community-it/kernel-it/src/test/java/org/neo4j/kernel/RecoveryIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,6 @@
import org.neo4j.graphdb.factory.GraphDatabaseBuilder.DatabaseCreator;
import org.neo4j.graphdb.factory.module.PlatformModule;
import org.neo4j.graphdb.mockfs.EphemeralFileSystemAbstraction;
import org.neo4j.helpers.collection.Iterables;
import org.neo4j.helpers.collection.Iterators;
import org.neo4j.io.ByteUnit;
import org.neo4j.io.fs.FileSystemAbstraction;
Expand All @@ -69,7 +68,6 @@
import org.neo4j.kernel.impl.api.index.inmemory.UpdateCapturingIndexProvider;
import org.neo4j.kernel.impl.factory.GraphDatabaseFacade;
import org.neo4j.kernel.impl.pagecache.ConfiguringPageCacheFactory;
import org.neo4j.kernel.impl.storageengine.impl.recordstorage.RecordStorageEngine;
import org.neo4j.kernel.impl.store.NeoStores;
import org.neo4j.kernel.impl.store.RecordStore;
import org.neo4j.kernel.impl.store.StoreFactory;
Expand Down Expand Up @@ -115,6 +113,7 @@
import static org.neo4j.graphdb.facade.GraphDatabaseDependencies.newDependencies;
import static org.neo4j.helpers.ArrayUtil.array;
import static org.neo4j.helpers.collection.Iterables.asList;
import static org.neo4j.helpers.collection.Iterables.count;
import static org.neo4j.kernel.configuration.Config.defaults;

public class RecoveryIT
Expand Down Expand Up @@ -147,11 +146,13 @@ public void idGeneratorsRebuildAfterRecovery() throws IOException
File restoreDbStoreDir = copyTransactionLogs();

GraphDatabaseService recoveredDatabase = startDatabase( restoreDbStoreDir );
NeoStores neoStore = ((GraphDatabaseAPI) recoveredDatabase).getDependencyResolver()
.resolveDependency( RecordStorageEngine.class ).testAccessNeoStores();
assertEquals( numberOfNodes, neoStore.getNodeStore().getHighId() );
// Make sure id generator has been rebuilt so this doesn't throw null pointer exception
assertTrue( neoStore.getNodeStore().nextId() > 0 );
try ( Transaction tx = recoveredDatabase.beginTx() )
{
assertEquals( numberOfNodes, count( recoveredDatabase.getAllNodes() ) );

// Make sure id generator has been rebuilt so this doesn't throw null pointer exception
recoveredDatabase.createNode();
}

database.shutdown();
recoveredDatabase.shutdown();
Expand All @@ -174,7 +175,7 @@ public void reportProgressOnRecovery() throws IOException
GraphDatabaseService recoveredDatabase = startDatabase( restoreDbStoreDir );
try ( Transaction transaction = recoveredDatabase.beginTx() )
{
assertEquals( 10, Iterables.count( recoveredDatabase.getAllNodes() ) );
assertEquals( 10, count( recoveredDatabase.getAllNodes() ) );
}
logProvider.assertContainsMessageContaining( "10% completed" );
logProvider.assertContainsMessageContaining( "100% completed" );
Expand Down
8 changes: 5 additions & 3 deletions ...-it/src/test/java/org/neo4j/kernel/impl/api/TransactionRepresentationCommitProcessIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
import org.neo4j.kernel.impl.index.DummyIndexExtensionFactory;
import org.neo4j.kernel.impl.storageengine.impl.recordstorage.RecordStorageEngine;
import org.neo4j.kernel.impl.store.NeoStores;
import org.neo4j.kernel.impl.transaction.log.TransactionIdStore;
import org.neo4j.kernel.impl.transaction.log.checkpoint.CheckPointer;
import org.neo4j.kernel.impl.transaction.log.checkpoint.SimpleTriggerInfo;
import org.neo4j.test.rule.DatabaseRule;
Expand Down Expand Up @@ -106,14 +107,15 @@ private void randomSleep()
done.set( true );
workers.awaitAndThrowOnError();

NeoStores neoStores = getDependency(RecordStorageEngine.class).testAccessNeoStores();
NeoStores neoStores = getDependency( RecordStorageEngine.class ).testAccessNeoStores();
assertThat( "Count store should be rotated once at least", neoStores.getCounts().txId(), greaterThan( 0L ) );

long lastRotationTx = getDependency( CheckPointer.class ).forceCheckPoint( new SimpleTriggerInfo( "test" ) );
TransactionIdStore txIdStore = getDependency( TransactionIdStore.class );
assertEquals( "NeoStore last closed transaction id should be equal last count store rotation transaction id.",
neoStores.getMetaDataStore().getLastClosedTransactionId(), lastRotationTx );
txIdStore.getLastClosedTransactionId(), lastRotationTx );
assertEquals( "Last closed transaction should be last rotated tx in count store",
neoStores.getMetaDataStore().getLastClosedTransactionId(), neoStores.getCounts().txId() );
txIdStore.getLastClosedTransactionId(), neoStores.getCounts().txId() );
}

private <T> T getDependency( Class<T> clazz )
Expand Down
34 changes: 7 additions & 27 deletions community/community-it/kernel-it/src/test/java/recovery/CountsStoreRecoveryTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,21 +25,20 @@
import org.junit.Test;

import java.io.IOException;
import java.util.concurrent.atomic.AtomicInteger;

import org.neo4j.graphdb.GraphDatabaseService;
import org.neo4j.graphdb.Transaction;
import org.neo4j.io.fs.FileSystemAbstraction;
import org.neo4j.kernel.impl.api.CountsVisitor;
import org.neo4j.kernel.impl.api.index.inmemory.InMemoryIndexProvider;
import org.neo4j.kernel.impl.api.index.inmemory.InMemoryIndexProviderFactory;
import org.neo4j.kernel.impl.storageengine.impl.recordstorage.RecordStorageEngine;
import org.neo4j.kernel.impl.store.MetaDataStore;
import org.neo4j.kernel.impl.store.NeoStores;
import org.neo4j.kernel.impl.store.counts.CountsTracker;
import org.neo4j.kernel.impl.transaction.log.checkpoint.CheckPointer;
import org.neo4j.kernel.impl.transaction.log.checkpoint.SimpleTriggerInfo;
import org.neo4j.kernel.internal.GraphDatabaseAPI;
import org.neo4j.storageengine.api.StorageEngine;
import org.neo4j.storageengine.api.StorageReader;
import org.neo4j.test.TestGraphDatabaseFactory;
import org.neo4j.test.rule.fs.EphemeralFileSystemRule;

Expand All @@ -62,24 +61,12 @@ public void shouldRecoverTheCountsStoreEvenWhenIfNeoStoreDoesNotNeedRecovery() t
crashAndRestart();

// then
final AtomicInteger number = new AtomicInteger( 0 );
counts().accept( new CountsVisitor.Adapter()
try ( StorageReader reader = ((GraphDatabaseAPI) db).getDependencyResolver().resolveDependency( StorageEngine.class ).newReader() )
{
@Override
public void visitNodeCount( int labelId, long count )
{
number.incrementAndGet();
if ( labelId != -1 )
{
assertEquals( 1, count );
}
else
{
assertEquals( 2, count );
}
}
} );
assertEquals( 3, number.get() );
assertEquals( 1, reader.countsForNode( reader.labelGetForName( "A" ) ) );
assertEquals( 1, reader.countsForNode( reader.labelGetForName( "B" ) ) );
assertEquals( 2, reader.countsForNode( -1 ) );
}
}

private void flushNeoStoreOnly()
Expand All @@ -90,13 +77,6 @@ private void flushNeoStoreOnly()
metaDataStore.flush();
}

private CountsTracker counts()
{
return ((GraphDatabaseAPI) db).getDependencyResolver()
.resolveDependency( RecordStorageEngine.class ).testAccessNeoStores()
.getCounts();
}

private void checkPoint() throws IOException
{
((GraphDatabaseAPI) db).getDependencyResolver()
Expand Down

0 comments on commit ae4c6d4

Please sign in to comment.