Skip to content

Commit

Permalink
Restore QUEUE_THRESHOLD, javadocs.
Browse files Browse the repository at this point in the history 
Remove unrelated code style changes, renaming stuff.
  • Loading branch information
@MishaDemianenko
MishaDemianenko committed Sep 27, 2016
1 parent 1aa2ef8 commit 9c0f29e
Show file tree
Hide file tree
Showing 17 changed files with 97 additions and 70 deletions.
4 changes: 2 additions & 2 deletions ...ernal/compiler/v3_0/planner/logical/cardinality/ExpressionSelectivityCalculatorTest.scala
View file
Expand Up @@ -20,8 +20,8 @@
package org.neo4j.cypher.internal.compiler.v3_0.planner.logical.cardinality

import org.mockito.Mockito.when
import org.neo4j.cypher.internal.compiler.v3_0.planner.logical.{Cardinality, Selectivity}
import org.neo4j.cypher.internal.compiler.v3_0.planner.logical.plans.IdName
import org.neo4j.cypher.internal.compiler.v3_0.planner.logical.{Cardinality, Selectivity}
import org.neo4j.cypher.internal.compiler.v3_0.planner.{Predicate, Selections}
import org.neo4j.cypher.internal.compiler.v3_0.spi.GraphStatistics
import org.neo4j.cypher.internal.frontend.v3_0.ast._
Expand Down Expand Up @@ -152,6 +152,6 @@ class ExpressionSelectivityCalculatorTest extends CypherFunSuite with AstConstru
implicit val selections = mock[Selections]

val expr = HasLabels(null, Seq(LabelName("Foo")(pos)))(pos)
// calculator(expr) should equal(Selectivity(1.0 / 10.0))
calculator(expr) should equal(Selectivity(1.0 / 10.0))
}
}
9 changes: 7 additions & 2 deletions community/kernel/src/main/java/org/neo4j/kernel/api/index/IndexPopulator.java
View file
Expand Up @@ -110,7 +110,12 @@ void add( Collection<NodePropertyUpdate> updates )
*/
void includeSample( NodePropertyUpdate update );

void configureSampling( boolean fullIndexSampling );
/**
* Configure specific type of sampling that should be used during index population.
* Depends from type of node scan that is used during index population
* @param onlineSampling should online (sampling based on index population and updates) be used
*/
void configureSampling( boolean onlineSampling );

IndexSample sampleResult();

Expand Down Expand Up @@ -158,7 +163,7 @@ public void includeSample( NodePropertyUpdate update )
}

@Override
public void configureSampling( boolean fullIndexSampling )
public void configureSampling( boolean onlineSampling )
{
}

Expand Down
2 changes: 1 addition & 1 deletion .../kernel/src/main/java/org/neo4j/kernel/impl/api/index/BatchingMultipleIndexPopulator.java
View file
Expand Up @@ -77,7 +77,7 @@ public class BatchingMultipleIndexPopulator extends MultipleIndexPopulator
private static final String EOL = System.lineSeparator();
private static final String FLUSH_THREAD_NAME_PREFIX = "Index Population Flush Thread";

private final int QUEUE_THRESHOLD = FeatureToggles.getInteger( getClass(), QUEUE_THRESHOLD_NAME, 1 );
private final int QUEUE_THRESHOLD = FeatureToggles.getInteger( getClass(), QUEUE_THRESHOLD_NAME, 20_000 );
private final int TASK_QUEUE_SIZE = FeatureToggles.getInteger( getClass(), TASK_QUEUE_SIZE_NAME,
getNumberOfPopulationWorkers() * 2 );
private final int AWAIT_TIMEOUT_MINUTES = FeatureToggles.getInteger( getClass(), AWAIT_TIMEOUT_MINUTES_NAME, 30 );
Expand Down
2 changes: 1 addition & 1 deletion community/kernel/src/main/java/org/neo4j/kernel/impl/api/index/MultipleIndexPopulator.java
View file
Expand Up @@ -270,7 +270,7 @@ public void includeSample( NodePropertyUpdate update )
}

@Override
public void configureSampling( boolean fullIndexSampling )
public void configureSampling( boolean onlineSampling )
{
throw new UnsupportedOperationException( "Multiple index populator can't be configured." );
}
Expand Down
4 changes: 2 additions & 2 deletions ...main/java/org/neo4j/kernel/impl/storageengine/impl/recordstorage/RecordStorageEngine.java
View file
Expand Up @@ -96,7 +96,7 @@
import org.neo4j.kernel.impl.transaction.state.RelationshipDeleter;
import org.neo4j.kernel.impl.transaction.state.RelationshipGroupGetter;
import org.neo4j.kernel.impl.transaction.state.TransactionRecordState;
import org.neo4j.kernel.impl.transaction.state.storeview.AdaptableIndexStoreView;
import org.neo4j.kernel.impl.transaction.state.storeview.DynamicIndexStoreView;
import org.neo4j.kernel.impl.transaction.state.storeview.NeoStoreIndexStoreView;
import org.neo4j.kernel.impl.util.DependencySatisfier;
import org.neo4j.kernel.impl.util.IdOrderingQueue;
Expand Down Expand Up @@ -216,7 +216,7 @@ public RecordStorageEngine(
labelScanStore = labelScanStoreProvider.getLabelScanStore();

schemaIndexProviderMap = new DefaultSchemaIndexProviderMap( indexProvider );
indexStoreView = new AdaptableIndexStoreView( labelScanStore, lockService, neoStores );
indexStoreView = new DynamicIndexStoreView( labelScanStore, lockService, neoStores );
indexingService = IndexingServiceFactory.createIndexingService( config, scheduler, schemaIndexProviderMap,
indexStoreView, tokenNameLookup,
Iterators.asList( new SchemaStorage( neoStores.getSchemaStore() ).allIndexRules() ), logProvider,
Expand Down
13 changes: 9 additions & 4 deletions ...te/storeview/AdaptableIndexStoreView.java → ...tate/storeview/DynamicIndexStoreView.java
View file
Expand Up @@ -39,15 +39,15 @@
* Store view that will try to use label scan store {@link LabelScanStore} for cases when estimated number of nodes
* is bellow certain threshold otherwise will fallback to whole store scan
*/
public class AdaptableIndexStoreView extends NeoStoreIndexStoreView
public class DynamicIndexStoreView extends NeoStoreIndexStoreView
{
private static final int VISIT_ALL_NODES_THRESHOLD_PERCENTAGE =
FeatureToggles.getInteger( AdaptableIndexStoreView.class, "all.nodes.visit.percentage.threshold", 50 );
FeatureToggles.getInteger( DynamicIndexStoreView.class, "all.nodes.visit.percentage.threshold", 50 );

private final LabelScanStore labelScanStore;
private final CountsTracker counts;

public AdaptableIndexStoreView( LabelScanStore labelScanStore, LockService locks, NeoStores neoStores )
public DynamicIndexStoreView( LabelScanStore labelScanStore, LockService locks, NeoStores neoStores )
{
super( locks, neoStores );
this.counts = neoStores.getCounts();
Expand All @@ -59,14 +59,19 @@ public <FAILURE extends Exception> StoreScan<FAILURE> visitNodes( int[] labelIds
IntPredicate propertyKeyIdFilter, Visitor<NodePropertyUpdates,FAILURE> propertyUpdatesVisitor,
Visitor<NodeLabelUpdate,FAILURE> labelUpdateVisitor )
{
if ( ArrayUtils.isEmpty( labelIds ) || isEmptyLabelScanStore() || isNumberOfLabeledNodesExceedThreshold( labelIds ) )
if ( useAllNodeStoreScan( labelIds ) )
{
return super.visitNodes( labelIds, propertyKeyIdFilter, propertyUpdatesVisitor, labelUpdateVisitor );
}
return new LabelScanViewNodeStoreScan<>( this, nodeStore, locks, propertyStore, labelScanStore, labelUpdateVisitor,
propertyUpdatesVisitor, labelIds, propertyKeyIdFilter );
}

private boolean useAllNodeStoreScan( int[] labelIds )
{
return ArrayUtils.isEmpty( labelIds ) || isEmptyLabelScanStore() || isNumberOfLabeledNodesExceedThreshold( labelIds );
}

private boolean isEmptyLabelScanStore()
{
return labelScanStore.allNodeLabelRanges().maxCount() == 0;
Expand Down
2 changes: 1 addition & 1 deletion community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/IndexCRUDIT.java
View file
Expand Up @@ -271,7 +271,7 @@ public void includeSample( NodePropertyUpdate update )
}

@Override
public void configureSampling( boolean fullIndexSampling )
public void configureSampling( boolean onlineSampling )
{
// nothing
}
Expand Down
2 changes: 0 additions & 2 deletions community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/IndexingServiceTest.java
View file
Expand Up @@ -251,13 +251,11 @@ public void shouldDeliverUpdatesThatOccurDuringPopulationToPopulator() throws Ex
order.verify( populator ).includeSample( add( 1, "value1" ) );
order.verify( populator ).add( Mockito.anyListOf (NodePropertyUpdate.class));


// invoked from indexAllNodes(), empty because the id we added (2) is bigger than the one we indexed (1)
//
// (We don't get an update for value2 here because we mock a fake store that doesn't contain it
// just for the purpose of testing this behavior)
order.verify( populator ).newPopulatingUpdater( storeView );

order.verify( updater ).close();
order.verify( populator ).verifyDeferredConstraints( storeView );
order.verify( populator ).sampleResult();
Expand Down
2 changes: 1 addition & 1 deletion community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/inmemory/InMemoryIndex.java
View file
Expand Up @@ -179,7 +179,7 @@ public void includeSample( NodePropertyUpdate update )
}

@Override
public void configureSampling( boolean fullIndexSampling )
public void configureSampling( boolean onlineSampling )
{
//nothing
}
Expand Down
25 changes: 20 additions & 5 deletions ...toreview/AdaptableIndexStoreViewTest.java → .../storeview/DynamicIndexStoreViewTest.java
View file
Expand Up @@ -19,7 +19,9 @@
*/
package org.neo4j.kernel.impl.transaction.state.storeview;

import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.mockito.Mockito;

Expand All @@ -42,6 +44,7 @@
import org.neo4j.register.Register;
import org.neo4j.register.Registers;
import org.neo4j.storageengine.api.schema.LabelScanReader;
import org.neo4j.unsafe.impl.internal.dragons.FeatureToggles;

import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyLong;
Expand All @@ -50,7 +53,7 @@
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.when;

public class AdaptableIndexStoreViewTest
public class DynamicIndexStoreViewTest
{

private LabelScanStore labelScanStore = mock( LabelScanStore.class );
Expand All @@ -62,6 +65,18 @@ public class AdaptableIndexStoreViewTest
private IntPredicate propertyKeyIdFilter = mock( IntPredicate.class );
private AllEntriesLabelScanReader nodeLabelRanges = mock( AllEntriesLabelScanReader.class );

@BeforeClass
public static void init()
{
FeatureToggles.set( DynamicIndexStoreView.class, "use.label.index", true );
}

@AfterClass
public static void cleanup()
{
FeatureToggles.set( DynamicIndexStoreView.class, "use.label.index", false );
}

@Before
public void setUp()
{
Expand All @@ -83,8 +98,8 @@ public void visitAllNodesWhenThresholdReached() throws Exception
mockLabelNodeCount( countStore, 2 );
mockLabelNodeCount( countStore, 3 );

AdaptableIndexStoreView storeView =
new AdaptableIndexStoreView( labelScanStore, LockService.NO_LOCK_SERVICE, neoStores );
DynamicIndexStoreView storeView =
new DynamicIndexStoreView( labelScanStore, LockService.NO_LOCK_SERVICE, neoStores );

StoreScan<Exception> storeScan = storeView
.visitNodes( new int[]{1, 2, 3}, propertyKeyIdFilter, propertyUpdateVisitor, labelUpdateVisitor );
Expand All @@ -110,8 +125,8 @@ public void visitOnlyLabeledNodesWhenThresholdNotReached() throws Exception
mockLabelNodeCount( countStore, 2 );
mockLabelNodeCount( countStore, 6 );

AdaptableIndexStoreView storeView =
new AdaptableIndexStoreView( labelScanStore, LockService.NO_LOCK_SERVICE, neoStores );
DynamicIndexStoreView storeView =
new DynamicIndexStoreView( labelScanStore, LockService.NO_LOCK_SERVICE, neoStores );

StoreScan<Exception> storeScan = storeView
.visitNodes( new int[]{2, 6}, propertyKeyIdFilter, propertyUpdateVisitor, labelUpdateVisitor );
Expand Down
2 changes: 1 addition & 1 deletion ...ava/org/neo4j/kernel/api/impl/labelscan/reader/PartitionedLuceneLabelScanStoreReader.java
View file
Expand Up @@ -44,7 +44,7 @@ public class PartitionedLuceneLabelScanStoreReader implements LabelScanReader
private final List<LabelScanReader> storeReaders;

public PartitionedLuceneLabelScanStoreReader( List<PartitionSearcher> searchers,
LabelScanStorageStrategy storageStrategy)
LabelScanStorageStrategy storageStrategy )
{
this( searchers.stream()
.map( searcher -> new SimpleLuceneLabelScanStoreReader( searcher, storageStrategy ) )
Expand Down
1 change: 0 additions & 1 deletion ...4j/kernel/api/impl/labelscan/storestrategy/NodeRangeDocumentLabelScanStorageStrategy.java
View file
Expand Up @@ -150,5 +150,4 @@ else if ( topDocs.scoreDocs.length > 1 )
throw new RuntimeException( e );
}
}

}
6 changes: 3 additions & 3 deletions ...c/main/java/org/neo4j/kernel/api/impl/schema/populator/NonUniqueLuceneIndexPopulator.java
View file
Expand Up @@ -49,10 +49,10 @@ public NonUniqueLuceneIndexPopulator( SchemaIndex luceneIndex, IndexSamplingConf
}

@Override
public void configureSampling( boolean fullIndexSampling )
public void configureSampling( boolean onlineSampling )
{
this.updateSampling = fullIndexSampling;
this.sampler = fullIndexSampling ? createDefaultSampler()
this.updateSampling = onlineSampling;
this.sampler = onlineSampling ? createDefaultSampler()
: new DirectNonUniqueIndexSampler( luceneIndex );
}

Expand Down
2 changes: 1 addition & 1 deletion .../src/main/java/org/neo4j/kernel/api/impl/schema/populator/UniqueLuceneIndexPopulator.java
View file
Expand Up @@ -67,7 +67,7 @@ public void includeSample( NodePropertyUpdate update )
}

@Override
public void configureSampling( boolean fullIndexSampling )
public void configureSampling( boolean onlineSampling )
{
// nothing to configure so far
}
Expand Down
18 changes: 9 additions & 9 deletions community/neo4j/src/test/java/schema/MultiIndexPopulationConcurrentUpdatesIT.java
View file
Expand Up @@ -65,7 +65,7 @@
import org.neo4j.kernel.impl.store.record.IndexRule;
import org.neo4j.kernel.impl.store.record.NodeRecord;
import org.neo4j.kernel.impl.transaction.state.DefaultSchemaIndexProviderMap;
import org.neo4j.kernel.impl.transaction.state.storeview.AdaptableIndexStoreView;
import org.neo4j.kernel.impl.transaction.state.storeview.DynamicIndexStoreView;
import org.neo4j.kernel.impl.transaction.state.storeview.LabelScanViewNodeStoreScan;
import org.neo4j.kernel.impl.transaction.state.storeview.NeoStoreIndexStoreView;
import org.neo4j.kernel.impl.util.JobScheduler;
Expand Down Expand Up @@ -216,8 +216,8 @@ private void launchCustomIndexPopulation( Map<String,Integer> labelNameIdMap, in
{
Statement statement = transactionStatementContextBridge.get();

AdaptableIndexStoreView storeView =
new AdaptableIndexStoreViewWrapper( labelScanStore, LockService.NO_LOCK_SERVICE, neoStores,
DynamicIndexStoreView storeView =
new DynamicIndexStoreViewWrapper( labelScanStore, LockService.NO_LOCK_SERVICE, neoStores,
updates );

SchemaIndexProviderMap providerMap = new DefaultSchemaIndexProviderMap( getSchemaIndexProvider() );
Expand Down Expand Up @@ -363,11 +363,11 @@ private JobScheduler getJobScheduler()
return embeddedDatabase.getDependencyResolver().resolveDependency( JobScheduler.class );
}

private class AdaptableIndexStoreViewWrapper extends AdaptableIndexStoreView
private class DynamicIndexStoreViewWrapper extends DynamicIndexStoreView
{
private List<NodePropertyUpdate> updates;

AdaptableIndexStoreViewWrapper( LabelScanStore labelScanStore, LockService locks, NeoStores neoStores,
DynamicIndexStoreViewWrapper( LabelScanStore labelScanStore, LockService locks, NeoStores neoStores,
List<NodePropertyUpdate> updates )
{
super( labelScanStore, locks, neoStores );
Expand All @@ -392,15 +392,15 @@ public <FAILURE extends Exception> StoreScan<FAILURE> visitNodes( int[] labelIds
private class LabelScanViewNodeStoreWrapper extends LabelScanViewNodeStoreScan
{
private LabelScanViewNodeStoreScan delegate;
private AdaptableIndexStoreViewWrapper adaptableIndexStoreViewWrapper;
private DynamicIndexStoreViewWrapper adaptableIndexStoreViewWrapper;
private List<NodePropertyUpdate> updates;

public LabelScanViewNodeStoreWrapper( NeoStoreIndexStoreView storeView, NodeStore nodeStore, LockService locks,
PropertyStore propertyStore,
LabelScanStore labelScanStore, Visitor labelUpdateVisitor,
Visitor propertyUpdatesVisitor, int[] labelIds, IntPredicate propertyKeyIdFilter,
LabelScanViewNodeStoreScan delegate,
AdaptableIndexStoreViewWrapper adaptableIndexStoreViewWrapper,
DynamicIndexStoreViewWrapper adaptableIndexStoreViewWrapper,
List<NodePropertyUpdate> updates )
{
super( storeView, nodeStore, locks, propertyStore, labelScanStore, labelUpdateVisitor,
Expand Down Expand Up @@ -428,15 +428,15 @@ public PrimitiveLongResourceIterator getNodeIdIterator()
private class DelegatingPrimitiveLongResourceIterator implements PrimitiveLongResourceIterator
{

private AdaptableIndexStoreViewWrapper adaptableIndexStoreViewWrapper;
private DynamicIndexStoreViewWrapper adaptableIndexStoreViewWrapper;
private List<NodePropertyUpdate> updates;
private LabelScanViewNodeStoreWrapper storeScan;
private PrimitiveLongResourceIterator delegate;

DelegatingPrimitiveLongResourceIterator(
LabelScanViewNodeStoreWrapper storeScan,
PrimitiveLongResourceIterator delegate,
AdaptableIndexStoreViewWrapper adaptableIndexStoreViewWrapper,
DynamicIndexStoreViewWrapper adaptableIndexStoreViewWrapper,
List<NodePropertyUpdate> updates )
{
this.storeScan = storeScan;
Expand Down
4 changes: 2 additions & 2 deletions enterprise/ha/src/test/java/org/neo4j/kernel/api/SchemaIndexHaIT.java
View file
Expand Up @@ -478,9 +478,9 @@ public void includeSample( NodePropertyUpdate update )
}

@Override
public void configureSampling( boolean fullIndexSampling )
public void configureSampling( boolean onlineSampling )
{
delegate.configureSampling( fullIndexSampling );
delegate.configureSampling( onlineSampling );
}

@Override
Expand Down

0 comments on commit 9c0f29e

Please sign in to comment.