Include updates from population queue into sampling.

Test set queue batch size into 1 sinse anything bigger then that does not work currently and test invariant of 12 missing updates as max does not hold in that case.
neo4j · Jun 27, 2018 · 373b66e · 373b66e
1 parent b4519d8
commit 373b66e
Show file tree

Hide file tree

Showing 4 changed files with 49 additions and 3 deletions.
diff --git a/...unity-it/kernel-it/src/test/java/org/neo4j/kernel/impl/api/index/IndexStatisticsTest.java b/...unity-it/kernel-it/src/test/java/org/neo4j/kernel/impl/api/index/IndexStatisticsTest.java
@@ -19,6 +19,7 @@
  */
 package org.neo4j.kernel.impl.api.index;
 
+import org.junit.After;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
@@ -65,6 +66,7 @@
 import org.neo4j.register.Registers;
 import org.neo4j.test.rule.DatabaseRule;
 import org.neo4j.test.rule.EmbeddedDatabaseRule;
+import org.neo4j.util.FeatureToggles;
 import org.neo4j.values.storable.Values;
 
 import static org.junit.Assert.assertEquals;
@@ -136,6 +138,7 @@ public static Object[] multiThreadedIndexPopulationEnabledValues()
     @Before
     public void before()
     {
+        FeatureToggles.set( MultipleIndexPopulator.class, MultipleIndexPopulator.QUEUE_THRESHOLD_NAME, 1 );
         GraphDatabaseAPI graphDatabaseAPI = dbRule.getGraphDatabaseAPI();
         this.db = graphDatabaseAPI;
         DependencyResolver dependencyResolver = graphDatabaseAPI.getDependencyResolver();
@@ -145,6 +148,12 @@ public void before()
                 .addMonitorListener( indexOnlineMonitor );
     }
 
+    @After
+    public void tearDown()
+    {
+        FeatureToggles.clear( MultipleIndexPopulator.class, MultipleIndexPopulator.QUEUE_THRESHOLD_NAME );
+    }
+
     @Test
     public void shouldProvideIndexStatisticsForDataCreatedWhenPopulationBeforeTheIndexIsOnline() throws KernelException
     {

diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/api/index/MultipleIndexPopulator.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/api/index/MultipleIndexPopulator.java
@@ -454,6 +454,7 @@ public void process( IndexEntryUpdate<?> update )
 
                 try
                 {
+                    population.populator.includeSample( update );
                     updater.process( update );
                 }
                 catch ( Throwable t )

diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NativeIndexPopulator.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NativeIndexPopulator.java
@@ -370,16 +370,53 @@ public void includeSample( IndexEntryUpdate<?> update )
         switch ( descriptor.type() )
         {
         case GENERAL:
-            nonUniqueSampler.include( SamplingUtil.encodedStringValuesForSampling( (Object[]) update.values() ) );
+            updateNonUniqueSample( update );
             break;
         case UNIQUE:
-            uniqueSampler.increment( 1 );
+            updateUniqueSample( update );
             break;
         default:
             throw new IllegalArgumentException( "Unexpected index type " + descriptor.type() );
         }
     }
 
+    private void updateUniqueSample( IndexEntryUpdate<?> update )
+    {
+        switch ( update.updateMode() )
+        {
+        case ADDED:
+            uniqueSampler.increment( 1 );
+            break;
+        case REMOVED:
+            uniqueSampler.increment( -1 );
+            break;
+        case CHANGED:
+            break;
+        default:
+            throw new IllegalArgumentException( "Unsupported update mode type:" + update.updateMode() );
+        }
+    }
+
+    private void updateNonUniqueSample( IndexEntryUpdate<?> update )
+    {
+        String encodedValues = SamplingUtil.encodedStringValuesForSampling( (Object[]) update.values() );
+        switch ( update.updateMode() )
+        {
+        case ADDED:
+            nonUniqueSampler.include( encodedValues );
+            break;
+        case REMOVED:
+            nonUniqueSampler.exclude( encodedValues );
+            break;
+        case CHANGED:
+            nonUniqueSampler.exclude( SamplingUtil.encodedStringValuesForSampling( (Object[]) update.beforeValues() ) );
+            nonUniqueSampler.include( encodedValues );
+            break;
+        default:
+            throw new IllegalArgumentException( "Unsupported update mode type:" + update.updateMode() );
+        }
+    }
+
     @Override
     public IndexSample sampleResult()
     {

diff --git a/...c/main/java/org/neo4j/kernel/impl/transaction/state/storeview/StoreViewNodeStoreScan.java b/...c/main/java/org/neo4j/kernel/impl/transaction/state/storeview/StoreViewNodeStoreScan.java
@@ -72,7 +72,6 @@ public void process( NodeRecord node ) throws FAILURE
         if ( propertyUpdatesVisitor != null && containsAnyEntityToken( labelIds, labels ) )
         {
             // Notify the property update visitor
-            // TODO: reuse object instead? Better in terms of speed and GC?
             EntityUpdates.Builder updates = EntityUpdates.forEntity( node.getId() ).withTokens( labels );
 
             if ( hasRelevantProperty(node, updates) )