Skip to content

Commit

Permalink
Skip scanning of empty database for count store initialization
Browse files Browse the repository at this point in the history 
Count store initialization is the most expensive part of empty database
start. At the same time, it's also useless since there is nothing to scan
and populate count store with.
  • Loading branch information
@MishaDemianenko
MishaDemianenko committed Jun 26, 2018
1 parent 8b59e63 commit 59998d0
Show file tree
Hide file tree
Showing 2 changed files with 101 additions and 40 deletions.
102 changes: 81 additions & 21 deletions ...ity-it/kernel-it/src/test/java/org/neo4j/kernel/impl/store/counts/CountsComputerTest.java
View file
Expand Up @@ -48,6 +48,8 @@
import org.neo4j.kernel.impl.store.id.DefaultIdGeneratorFactory;
import org.neo4j.kernel.impl.store.id.IdGeneratorFactory;
import org.neo4j.kernel.impl.transaction.log.TransactionIdStore;
import org.neo4j.kernel.impl.util.monitoring.ProgressReporter;
import org.neo4j.kernel.impl.util.monitoring.SilentProgressReporter;
import org.neo4j.kernel.internal.GraphDatabaseAPI;
import org.neo4j.kernel.lifecycle.Lifespan;
import org.neo4j.logging.NullLogProvider;
Expand All @@ -60,12 +62,15 @@
import org.neo4j.unsafe.impl.batchimport.cache.NumberArrayFactory;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.neo4j.kernel.impl.store.counts.keys.CountsKeyFactory.nodeKey;
import static org.neo4j.kernel.impl.store.counts.keys.CountsKeyFactory.relationshipKey;
import static org.neo4j.kernel.impl.transaction.log.TransactionIdStore.BASE_TX_ID;

public class CountsComputerTest
{
private static final String COUNTS_STORE_BASE = MetaDataStore.DEFAULT_NAME + StoreFactory.COUNTS_STORE;
private static final NullLogProvider LOG_PROVIDER = NullLogProvider.getInstance();
private static final Config CONFIG = Config.defaults();
private final EphemeralFileSystemRule fsRule = new EphemeralFileSystemRule();
Expand All @@ -80,6 +85,31 @@ public class CountsComputerTest
private GraphDatabaseBuilder dbBuilder;
private PageCache pageCache;

@Before
public void setup()
{
fs = fsRule.get();
dir = testDir.directory( "dir" ).getAbsoluteFile();
dbBuilder = new TestGraphDatabaseFactory().setFileSystem( new UncloseableDelegatingFileSystemAbstraction( fs ) )
.newImpermanentDatabaseBuilder( dir );
pageCache = pcRule.getPageCache( fs );
}

@Test
public void skipPopulationWhenNodeAndRelationshipStoresAreEmpty()
{
GraphDatabaseAPI db = (GraphDatabaseAPI) dbBuilder.newGraphDatabase();
long lastCommittedTransactionId = getLastTxId( db );
db.shutdown();

InvocationTrackingProgressReporter progressReporter = new InvocationTrackingProgressReporter();
rebuildCounts( lastCommittedTransactionId, progressReporter );

checkEmptyCountStore();
assertTrue( progressReporter.isCompleteInvoked() );
assertFalse( progressReporter.isStartInvoked() );
}

@Test
public void shouldCreateAnEmptyCountsStoreFromAnEmptyDatabase()
{
Expand All @@ -90,13 +120,7 @@ public void shouldCreateAnEmptyCountsStoreFromAnEmptyDatabase()

rebuildCounts( lastCommittedTransactionId );

try ( Lifespan life = new Lifespan() )
{
CountsTracker store = life.add( createCountsTracker() );
// a transaction for creating the label and a transaction for the node
assertEquals( BASE_TX_ID, store.txId() );
assertEquals( 0, store.totalEntriesStored() );
}
checkEmptyCountStore();
}

@Test
Expand Down Expand Up @@ -279,18 +303,6 @@ public void shouldCreateACountStoreWhenDBContainsDenseNodes()
}
}

@Before
public void setup()
{
fs = fsRule.get();
dir = testDir.directory( "dir" ).getAbsoluteFile();
dbBuilder = new TestGraphDatabaseFactory().setFileSystem( new UncloseableDelegatingFileSystemAbstraction( fs ) )
.newImpermanentDatabaseBuilder( dir );
pageCache = pcRule.getPageCache( fs );
}

private static final String COUNTS_STORE_BASE = MetaDataStore.DEFAULT_NAME + StoreFactory.COUNTS_STORE;

private File alphaStoreFile()
{
return new File( dir, COUNTS_STORE_BASE + CountsTracker.LEFT );
Expand All @@ -304,7 +316,16 @@ private File betaStoreFile()
private long getLastTxId( @SuppressWarnings( "deprecation" ) GraphDatabaseAPI db )
{
return db.getDependencyResolver().resolveDependency( TransactionIdStore.class ).getLastCommittedTransactionId();
}

private void checkEmptyCountStore()
{
try ( Lifespan life = new Lifespan() )
{
CountsTracker store = life.add( createCountsTracker() );
assertEquals( BASE_TX_ID, store.txId() );
assertEquals( 0, store.totalEntriesStored() );
}
}

private void cleanupCountsForRebuilding()
Expand All @@ -320,6 +341,11 @@ private CountsTracker createCountsTracker()
}

private void rebuildCounts( long lastCommittedTransactionId )
{
rebuildCounts( lastCommittedTransactionId, SilentProgressReporter.INSTANCE );
}

private void rebuildCounts( long lastCommittedTransactionId, ProgressReporter progressReporter )
{
cleanupCountsForRebuilding();

Expand All @@ -333,8 +359,8 @@ private void rebuildCounts( long lastCommittedTransactionId )
int highLabelId = (int) neoStores.getLabelTokenStore().getHighId();
int highRelationshipTypeId = (int) neoStores.getRelationshipTypeTokenStore().getHighId();
CountsComputer countsComputer = new CountsComputer(
lastCommittedTransactionId, nodeStore, relationshipStore, highLabelId, highRelationshipTypeId,
NumberArrayFactory.AUTO_WITHOUT_PAGECACHE );
lastCommittedTransactionId, nodeStore, relationshipStore, highLabelId, highRelationshipTypeId, NumberArrayFactory.AUTO_WITHOUT_PAGECACHE,
progressReporter );
CountsTracker countsTracker = createCountsTracker();
life.add( countsTracker.setInitializer( countsComputer ) );
}
Expand All @@ -346,4 +372,38 @@ private long get( CountsTracker store, CountsKey key )
store.get( key, value );
return value.readSecond();
}

private static class InvocationTrackingProgressReporter implements ProgressReporter
{
private boolean startInvoked;
private boolean completeInvoked;

@Override
public void start( long max )
{
startInvoked = true;
}

@Override
public void progress( long add )
{

}

@Override
public void completed()
{
completeInvoked = true;
}

boolean isStartInvoked()
{
return startInvoked;
}

boolean isCompleteInvoked()
{
return completeInvoked;
}
}
}
39 changes: 20 additions & 19 deletions community/kernel/src/main/java/org/neo4j/kernel/impl/store/CountsComputer.java
View file
Expand Up @@ -35,9 +35,6 @@

public class CountsComputer implements DataInitializer<CountsAccessor.Updater>
{

private final NumberArrayFactory numberArrayFactory;

public static void recomputeCounts( NeoStores stores, PageCache pageCache )
{
MetaDataStore metaDataStore = stores.getMetaDataStore();
Expand All @@ -48,14 +45,16 @@ public static void recomputeCounts( NeoStores stores, PageCache pageCache )
}
}

private static final int EMPTY_NODE_AND_RELATIONSHIP_STORE = -2;
private final NodeStore nodes;
private final RelationshipStore relationships;
private final int highLabelId;
private final int highRelationshipTypeId;
private final long lastCommittedTransactionId;
private final ProgressReporter progressMonitor;
private final NumberArrayFactory numberArrayFactory;

public CountsComputer( NeoStores stores, PageCache pageCache )
CountsComputer( NeoStores stores, PageCache pageCache )
{
this( stores.getMetaDataStore().getLastCommittedTransactionId(),
stores.getNodeStore(), stores.getRelationshipStore(),
Expand All @@ -64,8 +63,8 @@ public CountsComputer( NeoStores stores, PageCache pageCache )
NumberArrayFactory.auto( pageCache, stores.getStoreDir(), true ) );
}

public CountsComputer( long lastCommittedTransactionId, NodeStore nodes, RelationshipStore relationships,
int highLabelId, int highRelationshipTypeId, NumberArrayFactory numberArrayFactory )
private CountsComputer( long lastCommittedTransactionId, NodeStore nodes, RelationshipStore relationships, int highLabelId, int highRelationshipTypeId,
NumberArrayFactory numberArrayFactory )
{
this( lastCommittedTransactionId, nodes, relationships, highLabelId, highRelationshipTypeId,
numberArrayFactory, SilentProgressReporter.INSTANCE );
Expand All @@ -86,23 +85,25 @@ public CountsComputer( long lastCommittedTransactionId, NodeStore nodes, Relatio
@Override
public void initialize( CountsAccessor.Updater countsUpdater )
{
progressMonitor.start( nodes.getHighestPossibleIdInUse() + relationships.getHighestPossibleIdInUse() );
NodeLabelsCache cache = new NodeLabelsCache( numberArrayFactory, highLabelId );
try
long recordsToVisit = Math.addExact( nodes.getHighestPossibleIdInUse(), relationships.getHighestPossibleIdInUse() );
if ( recordsToVisit != EMPTY_NODE_AND_RELATIONSHIP_STORE )
{
progressMonitor.start( recordsToVisit );
populateCountStore( countsUpdater );
}
progressMonitor.completed();
}

private void populateCountStore( CountsAccessor.Updater countsUpdater )
{
try ( NodeLabelsCache cache = new NodeLabelsCache( numberArrayFactory, highLabelId ) )
{
// Count nodes
superviseDynamicExecution(
new NodeCountsStage( Configuration.DEFAULT, cache, nodes, highLabelId, countsUpdater,
progressMonitor ) );
superviseDynamicExecution( new NodeCountsStage( Configuration.DEFAULT, cache, nodes, highLabelId, countsUpdater, progressMonitor ) );
// Count relationships
superviseDynamicExecution(
new RelationshipCountsStage( Configuration.DEFAULT, cache, relationships, highLabelId,
highRelationshipTypeId, countsUpdater, numberArrayFactory, progressMonitor ) );
}
finally
{
cache.close();
progressMonitor.completed();
new RelationshipCountsStage( Configuration.DEFAULT, cache, relationships, highLabelId, highRelationshipTypeId, countsUpdater,
numberArrayFactory, progressMonitor ) );
}
}

Expand Down

0 comments on commit 59998d0

Please sign in to comment.