Some basic plugging storing of settings into the populator/accessor

neo4j · Sep 5, 2018 · 4cf0242 · 4cf0242
1 parent 920f500
commit 4cf0242
Show file tree

Hide file tree

Showing 24 changed files with 218 additions and 55 deletions.
diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/GenericLayout.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/GenericLayout.java
@@ -139,4 +139,9 @@ public void minimalSplitter( CompositeGenericKey left, CompositeGenericKey right
     {
         CompositeGenericKey.minimalSplitter( left, right, into );
     }
+
+    IndexSpecificSpaceFillingCurveSettingsCache getSpaceFillingCurveSettings()
+    {
+        return spatialSettings;
+    }
 }
diff --git a/...y/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/GenericNativeIndexAccessor.java b/...y/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/GenericNativeIndexAccessor.java
@@ -28,6 +28,7 @@
 import org.neo4j.io.pagecache.PageCache;
 import org.neo4j.kernel.api.index.IndexProvider;
 import org.neo4j.kernel.impl.api.index.sampling.IndexSamplingConfig;
+import org.neo4j.kernel.impl.index.schema.config.IndexSpecificSpaceFillingCurveSettingsCache;
 import org.neo4j.kernel.impl.util.Validator;
 import org.neo4j.storageengine.api.schema.IndexReader;
 import org.neo4j.storageengine.api.schema.StoreIndexDescriptor;
@@ -39,9 +40,10 @@ class GenericNativeIndexAccessor extends NativeIndexAccessor<CompositeGenericKey
 
     GenericNativeIndexAccessor( PageCache pageCache, FileSystemAbstraction fs, File storeFile, IndexLayout<CompositeGenericKey,NativeIndexValue> layout,
             RecoveryCleanupWorkCollector recoveryCleanupWorkCollector, IndexProvider.Monitor monitor, StoreIndexDescriptor descriptor,
-            IndexSamplingConfig samplingConfig ) throws IOException
+            IndexSamplingConfig samplingConfig, IndexSpecificSpaceFillingCurveSettingsCache spaceFillingCurveSettings ) throws IOException
     {
-        super( pageCache, fs, storeFile, layout, recoveryCleanupWorkCollector, monitor, descriptor, samplingConfig );
+        super( pageCache, fs, storeFile, layout, recoveryCleanupWorkCollector, monitor, descriptor, samplingConfig,
+                new SpaceFillingCurveSettingsWriter( spaceFillingCurveSettings ) );
     }
 
     @Override

diff --git a/.../kernel/src/main/java/org/neo4j/kernel/impl/index/schema/GenericNativeIndexPopulator.java b/.../kernel/src/main/java/org/neo4j/kernel/impl/index/schema/GenericNativeIndexPopulator.java
@@ -25,15 +25,17 @@
 import org.neo4j.io.pagecache.PageCache;
 import org.neo4j.kernel.api.index.IndexProvider;
 import org.neo4j.kernel.impl.api.index.sampling.IndexSamplingConfig;
+import org.neo4j.kernel.impl.index.schema.config.IndexSpecificSpaceFillingCurveSettingsCache;
 import org.neo4j.storageengine.api.schema.IndexReader;
 import org.neo4j.storageengine.api.schema.StoreIndexDescriptor;
 
 class GenericNativeIndexPopulator extends NativeIndexPopulator<CompositeGenericKey,NativeIndexValue>
 {
     GenericNativeIndexPopulator( PageCache pageCache, FileSystemAbstraction fs, File storeFile, IndexLayout<CompositeGenericKey,NativeIndexValue> layout,
-            IndexProvider.Monitor monitor, StoreIndexDescriptor descriptor, IndexSamplingConfig samplingConfig )
+            IndexProvider.Monitor monitor, StoreIndexDescriptor descriptor, IndexSamplingConfig samplingConfig,
+            IndexSpecificSpaceFillingCurveSettingsCache spatialSettings )
     {
-        super( pageCache, fs, storeFile, layout, monitor, descriptor, samplingConfig );
+        super( pageCache, fs, storeFile, layout, monitor, descriptor, samplingConfig, new SpaceFillingCurveSettingsWriter( spatialSettings ) );
     }
 
     @Override

diff --git a/...y/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/GenericNativeIndexProvider.java b/...y/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/GenericNativeIndexProvider.java
@@ -21,9 +21,13 @@
 
 import java.io.File;
 import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.HashMap;
+import java.util.Map;
 
 import org.neo4j.gis.spatial.index.curves.SpaceFillingCurveConfiguration;
 import org.neo4j.graphdb.factory.GraphDatabaseSettings;
+import org.neo4j.index.internal.gbptree.GBPTree;
 import org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector;
 import org.neo4j.internal.kernel.api.IndexCapability;
 import org.neo4j.internal.kernel.api.IndexOrder;
@@ -37,7 +41,10 @@
 import org.neo4j.kernel.configuration.Config;
 import org.neo4j.kernel.impl.api.index.sampling.IndexSamplingConfig;
 import org.neo4j.kernel.impl.index.schema.config.ConfiguredSpaceFillingCurveSettingsCache;
+import org.neo4j.kernel.impl.index.schema.config.IndexSpecificSpaceFillingCurveSettingsCache;
+import org.neo4j.kernel.impl.index.schema.config.SpaceFillingCurveSettings;
 import org.neo4j.storageengine.api.schema.StoreIndexDescriptor;
+import org.neo4j.values.storable.CoordinateReferenceSystem;
 import org.neo4j.values.storable.ValueCategory;
 
 import static org.neo4j.kernel.impl.index.schema.SpatialIndexProvider.getConfiguredSpaceFillingCurveConfiguration;
@@ -136,25 +143,37 @@ public GenericNativeIndexProvider( int priority, IndexDirectoryStructure.Factory
     }
 
     @Override
-    IndexLayout<CompositeGenericKey,NativeIndexValue> layout( StoreIndexDescriptor descriptor )
+    IndexLayout<CompositeGenericKey,NativeIndexValue> layout( StoreIndexDescriptor descriptor, File storeFile )
     {
-        int numberOfSlots = descriptor.properties().length;
-        // TODO read header from the tree and build a IndexSpecificSpaceFillingCurveSettingsCache and pass in
-        return new GenericLayout( numberOfSlots, null );
+        try
+        {
+            int numberOfSlots = descriptor.properties().length;
+            Map<CoordinateReferenceSystem,SpaceFillingCurveSettings> settings = new HashMap<>();
+            GBPTree.readHeader( pageCache, storeFile, new SpaceFillingCurveSettingsReader( settings ) );
+            return new GenericLayout( numberOfSlots, new IndexSpecificSpaceFillingCurveSettingsCache( configuredSettings, settings ) );
+        }
+        catch ( IOException e )
+        {
+            throw new UncheckedIOException( e );
+        }
     }
 
     @Override
     protected IndexPopulator newIndexPopulator( File storeFile, IndexLayout<CompositeGenericKey,NativeIndexValue> layout, StoreIndexDescriptor descriptor,
             IndexSamplingConfig samplingConfig )
     {
-        return new GenericNativeIndexPopulator( pageCache, fs, storeFile, layout, monitor, descriptor, samplingConfig );
+        // TODO this layout cast isn't nice, try and get rid of it
+        return new GenericNativeIndexPopulator( pageCache, fs, storeFile, layout, monitor, descriptor, samplingConfig,
+                ((GenericLayout) layout).getSpaceFillingCurveSettings() );
     }
 
     @Override
     protected IndexAccessor newIndexAccessor( File storeFile, IndexLayout<CompositeGenericKey,NativeIndexValue> layout, StoreIndexDescriptor descriptor,
             IndexSamplingConfig samplingConfig ) throws IOException
     {
-        return new GenericNativeIndexAccessor( pageCache, fs, storeFile, layout, recoveryCleanupWorkCollector, monitor, descriptor, samplingConfig );
+        // TODO this layout cast isn't nice, try and get rid of it
+        return new GenericNativeIndexAccessor( pageCache, fs, storeFile, layout, recoveryCleanupWorkCollector, monitor, descriptor, samplingConfig,
+                ((GenericLayout) layout).getSpaceFillingCurveSettings() );
     }
 
     @Override

diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NativeIndexAccessor.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NativeIndexAccessor.java
@@ -22,13 +22,15 @@
 import java.io.File;
 import java.io.IOException;
 import java.io.UncheckedIOException;
+import java.util.function.Consumer;
 
 import org.neo4j.graphdb.ResourceIterator;
 import org.neo4j.helpers.collection.BoundedIterable;
 import org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector;
 import org.neo4j.io.fs.FileSystemAbstraction;
 import org.neo4j.io.pagecache.IOLimiter;
 import org.neo4j.io.pagecache.PageCache;
+import org.neo4j.io.pagecache.PageCursor;
 import org.neo4j.kernel.api.index.IndexAccessor;
 import org.neo4j.kernel.api.index.IndexProvider;
 import org.neo4j.kernel.api.index.NodePropertyAccessor;
@@ -39,22 +41,24 @@
 
 import static org.neo4j.helpers.collection.Iterators.asResourceIterator;
 import static org.neo4j.helpers.collection.Iterators.iterator;
-import static org.neo4j.index.internal.gbptree.GBPTree.NO_HEADER_WRITER;
+import static org.neo4j.kernel.impl.index.schema.NativeIndexPopulator.BYTE_ONLINE;
 
 public abstract class NativeIndexAccessor<KEY extends NativeIndexKey<KEY>, VALUE extends NativeIndexValue> extends NativeIndex<KEY,VALUE>
         implements IndexAccessor
 {
     private final NativeIndexUpdater<KEY,VALUE> singleUpdater;
     final IndexSamplingConfig samplingConfig;
+    private final NativeIndexHeaderWriter headerWriter;
 
     NativeIndexAccessor( PageCache pageCache, FileSystemAbstraction fs, File storeFile, IndexLayout<KEY,VALUE> layout,
             RecoveryCleanupWorkCollector recoveryCleanupWorkCollector, IndexProvider.Monitor monitor, StoreIndexDescriptor descriptor,
-            IndexSamplingConfig samplingConfig ) throws IOException
+            IndexSamplingConfig samplingConfig, Consumer<PageCursor> additionalHeaderWriter ) throws IOException
     {
         super( pageCache, fs, storeFile, layout, monitor, descriptor );
         singleUpdater = new NativeIndexUpdater<>( layout.newKey(), layout.newValue() );
         this.samplingConfig = samplingConfig;
-        instantiateTree( recoveryCleanupWorkCollector, NO_HEADER_WRITER );
+        instantiateTree( recoveryCleanupWorkCollector, additionalHeaderWriter );
+        headerWriter = new NativeIndexHeaderWriter( BYTE_ONLINE, additionalHeaderWriter );
     }
 
     @Override
@@ -88,7 +92,7 @@ public NativeIndexUpdater<KEY, VALUE> newUpdater( IndexUpdateMode mode )
     @Override
     public void force( IOLimiter ioLimiter )
     {
-        tree.checkpoint( ioLimiter );
+        tree.checkpoint( ioLimiter, headerWriter );
     }
 
     @Override

diff --git a/...nity/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NativeIndexHeaderWriter.java b/...nity/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NativeIndexHeaderWriter.java
@@ -30,15 +30,18 @@
 class NativeIndexHeaderWriter implements Consumer<PageCursor>
 {
     private final byte state;
+    private final Consumer<PageCursor> additionalHeaderWriter;
 
-    NativeIndexHeaderWriter( byte state )
+    NativeIndexHeaderWriter( byte state, Consumer<PageCursor> additionalHeaderWriter )
     {
         this.state = state;
+        this.additionalHeaderWriter = additionalHeaderWriter;
     }
 
     @Override
     public void accept( PageCursor cursor )
     {
         cursor.putByte( state );
+        additionalHeaderWriter.accept( cursor );
     }
 }
diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NativeIndexPopulator.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NativeIndexPopulator.java
@@ -73,6 +73,7 @@ public abstract class NativeIndexPopulator<KEY extends NativeIndexKey<KEY>, VALU
     private final VALUE treeValue;
     private final UniqueIndexSampler uniqueSampler;
     final IndexSamplingConfig samplingConfig;
+    private final Consumer<PageCursor> additionalHeaderWriter;
 
     private WorkSync<IndexUpdateApply<KEY,VALUE>,IndexUpdateWork<KEY,VALUE>> additionsWorkSync;
     private WorkSync<IndexUpdateApply<KEY,VALUE>,IndexUpdateWork<KEY,VALUE>> updatesWorkSync;
@@ -82,12 +83,13 @@ public abstract class NativeIndexPopulator<KEY extends NativeIndexKey<KEY>, VALU
     private boolean closed;
 
     NativeIndexPopulator( PageCache pageCache, FileSystemAbstraction fs, File storeFile, IndexLayout<KEY,VALUE> layout, IndexProvider.Monitor monitor,
-                                StoreIndexDescriptor descriptor, IndexSamplingConfig samplingConfig )
+            StoreIndexDescriptor descriptor, IndexSamplingConfig samplingConfig, Consumer<PageCursor> additionalHeaderWriter )
     {
         super( pageCache, fs, storeFile, layout, monitor, descriptor );
         this.treeKey = layout.newKey();
         this.treeValue = layout.newValue();
         this.samplingConfig = samplingConfig;
+        this.additionalHeaderWriter = additionalHeaderWriter;
         switch ( descriptor.type() )
         {
         case GENERAL:
@@ -109,7 +111,7 @@ public void clear()
     @Override
     public synchronized void create()
     {
-        create( new NativeIndexHeaderWriter( BYTE_POPULATING ) );
+        create( new NativeIndexHeaderWriter( BYTE_POPULATING, additionalHeaderWriter ) );
     }
 
     protected synchronized void create( Consumer<PageCursor> headerWriter )
@@ -299,7 +301,7 @@ private void markTreeAsFailed()
 
     void markTreeAsOnline()
     {
-        tree.checkpoint( IOLimiter.UNLIMITED, pc -> pc.putByte( BYTE_ONLINE ) );
+        tree.checkpoint( IOLimiter.UNLIMITED, new NativeIndexHeaderWriter( BYTE_ONLINE, additionalHeaderWriter ) );
     }
 
     static class IndexUpdateApply<KEY extends NativeIndexKey<KEY>, VALUE extends NativeIndexValue>

diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NativeIndexProvider.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NativeIndexProvider.java
@@ -62,7 +62,7 @@ protected NativeIndexProvider( IndexProviderDescriptor descriptor, int priority,
         this.readOnly = readOnly;
     }
 
-    abstract IndexLayout<KEY,VALUE> layout( StoreIndexDescriptor descriptor );
+    abstract IndexLayout<KEY,VALUE> layout( StoreIndexDescriptor descriptor, File storeFile );
 
     @Override
     public IndexPopulator getPopulator( StoreIndexDescriptor descriptor, IndexSamplingConfig samplingConfig )
@@ -73,7 +73,7 @@ public IndexPopulator getPopulator( StoreIndexDescriptor descriptor, IndexSampli
         }
 
         File storeFile = nativeIndexFileFromIndexId( descriptor.getId() );
-        return newIndexPopulator( storeFile, layout( descriptor ), descriptor, samplingConfig );
+        return newIndexPopulator( storeFile, layout( descriptor, storeFile ), descriptor, samplingConfig );
     }
 
     protected abstract IndexPopulator newIndexPopulator( File storeFile, IndexLayout<KEY,VALUE> layout, StoreIndexDescriptor descriptor,
@@ -83,7 +83,7 @@ protected abstract IndexPopulator newIndexPopulator( File storeFile, IndexLayout
     public IndexAccessor getOnlineAccessor( StoreIndexDescriptor descriptor, IndexSamplingConfig samplingConfig ) throws IOException
     {
         File storeFile = nativeIndexFileFromIndexId( descriptor.getId() );
-        return newIndexAccessor( storeFile, layout( descriptor ), descriptor, samplingConfig );
+        return newIndexAccessor( storeFile, layout( descriptor, storeFile ), descriptor, samplingConfig );
     }
 
     protected abstract IndexAccessor newIndexAccessor( File storeFile, IndexLayout<KEY,VALUE> layout, StoreIndexDescriptor descriptor,

diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NumberIndexAccessor.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NumberIndexAccessor.java
@@ -30,13 +30,15 @@
 import org.neo4j.storageengine.api.schema.IndexReader;
 import org.neo4j.storageengine.api.schema.StoreIndexDescriptor;
 
+import static org.neo4j.index.internal.gbptree.GBPTree.NO_HEADER_WRITER;
+
 public class NumberIndexAccessor extends NativeIndexAccessor<NumberIndexKey,NativeIndexValue>
 {
     NumberIndexAccessor( PageCache pageCache, FileSystemAbstraction fs, File storeFile, IndexLayout<NumberIndexKey,NativeIndexValue> layout,
             RecoveryCleanupWorkCollector recoveryCleanupWorkCollector, IndexProvider.Monitor monitor, StoreIndexDescriptor descriptor,
             IndexSamplingConfig samplingConfig ) throws IOException
     {
-        super( pageCache, fs, storeFile, layout, recoveryCleanupWorkCollector, monitor, descriptor, samplingConfig );
+        super( pageCache, fs, storeFile, layout, recoveryCleanupWorkCollector, monitor, descriptor, samplingConfig, NO_HEADER_WRITER );
     }
 
     @Override

diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NumberIndexPopulator.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NumberIndexPopulator.java
@@ -28,12 +28,14 @@
 import org.neo4j.storageengine.api.schema.IndexReader;
 import org.neo4j.storageengine.api.schema.StoreIndexDescriptor;
 
+import static org.neo4j.index.internal.gbptree.GBPTree.NO_HEADER_WRITER;
+
 class NumberIndexPopulator extends NativeIndexPopulator<NumberIndexKey,NativeIndexValue>
 {
     NumberIndexPopulator( PageCache pageCache, FileSystemAbstraction fs, File storeFile, IndexLayout<NumberIndexKey,NativeIndexValue> layout,
             IndexProvider.Monitor monitor, StoreIndexDescriptor descriptor, IndexSamplingConfig samplingConfig )
     {
-        super( pageCache, fs, storeFile, layout, monitor, descriptor, samplingConfig );
+        super( pageCache, fs, storeFile, layout, monitor, descriptor, samplingConfig, NO_HEADER_WRITER );
     }
 
     @Override

diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NumberIndexProvider.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NumberIndexProvider.java
@@ -54,7 +54,7 @@ public NumberIndexProvider( PageCache pageCache, FileSystemAbstraction fs,
     }
 
     @Override
-    IndexLayout<NumberIndexKey,NativeIndexValue> layout( StoreIndexDescriptor descriptor )
+    IndexLayout<NumberIndexKey,NativeIndexValue> layout( StoreIndexDescriptor descriptor, File storeFile )
     {
         // split like this due to legacy reasons, there are old stores out there with these different identifiers
         switch ( descriptor.type() )

diff --git a/...nel/src/main/java/org/neo4j/kernel/impl/index/schema/SpaceFillingCurveSettingsReader.java b/...nel/src/main/java/org/neo4j/kernel/impl/index/schema/SpaceFillingCurveSettingsReader.java
@@ -0,0 +1,43 @@
+/*
+ * Copyright (c) 2002-2018 "Neo4j,"
+ * Neo4j Sweden AB [http://neo4j.com]
+ *
+ * This file is part of Neo4j.
+ *
+ * Neo4j is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.neo4j.kernel.impl.index.schema;
+
+import java.nio.ByteBuffer;
+import java.util.Map;
+
+import org.neo4j.index.internal.gbptree.Header;
+import org.neo4j.kernel.impl.index.schema.config.SpaceFillingCurveSettings;
+import org.neo4j.values.storable.CoordinateReferenceSystem;
+
+class SpaceFillingCurveSettingsReader implements Header.Reader
+{
+    private final Map<CoordinateReferenceSystem,SpaceFillingCurveSettings> settings;
+
+    SpaceFillingCurveSettingsReader( Map<CoordinateReferenceSystem,SpaceFillingCurveSettings> settings )
+    {
+        this.settings = settings;
+    }
+
+    @Override
+    public void read( ByteBuffer headerBytes )
+    {
+        // TODO read into the map there
+    }
+}
diff --git a/...nel/src/main/java/org/neo4j/kernel/impl/index/schema/SpaceFillingCurveSettingsWriter.java b/...nel/src/main/java/org/neo4j/kernel/impl/index/schema/SpaceFillingCurveSettingsWriter.java
@@ -0,0 +1,41 @@
+/*
+ * Copyright (c) 2002-2018 "Neo4j,"
+ * Neo4j Sweden AB [http://neo4j.com]
+ *
+ * This file is part of Neo4j.
+ *
+ * Neo4j is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.neo4j.kernel.impl.index.schema;
+
+import java.util.function.Consumer;
+
+import org.neo4j.io.pagecache.PageCursor;
+import org.neo4j.kernel.impl.index.schema.config.IndexSpecificSpaceFillingCurveSettingsCache;
+
+class SpaceFillingCurveSettingsWriter implements Consumer<PageCursor>
+{
+    private final IndexSpecificSpaceFillingCurveSettingsCache settings;
+
+    SpaceFillingCurveSettingsWriter( IndexSpecificSpaceFillingCurveSettingsCache settings )
+    {
+        this.settings = settings;
+    }
+
+    @Override
+    public void accept( PageCursor cursor )
+    {
+        // TODO write those settings!
+    }
+}