Make indexes platform independent

The previous Base64 encoding could introduce new lines if the encoded
string was longer than a given number of bytes.  Since new line
encodings are platform dependent so were our indexes.  This change
fixes that by removing the new lines from the Base64 encoding and
provides a migration path from older versions of the database, i.e.,
indexes that might contain the problematic encoding as values will be
forcely rebuilt.

Bump store version up to '0.A.5' in order to fail when trying to use
stores from 2.2-M0{1,2,3}.

Small refactor around UpgredableDatabase: added method
hasCurrentVersion for checking if the store has the current and
avoiding using the NeoStoreUtil directly.

community/consistency-check/src/test/java/org/neo4j/consistency/checking/full/NodeCorrectlyIndexedCheckTest.java

@@ -19,13 +19,14 @@
  */
 package org.neo4j.consistency.checking.full;
 
+import org.junit.Test;
+
 import java.io.File;
 import java.io.IOException;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
-
-import org.junit.Test;
+import java.util.Set;
 
 import org.neo4j.collection.primitive.PrimitiveLongCollections;
 import org.neo4j.collection.primitive.PrimitiveLongIterator;
@@ -47,13 +48,11 @@
 import org.neo4j.register.Register;
 
 import static java.util.Arrays.asList;
-
 import static org.mockito.Matchers.any;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.verifyZeroInteractions;
 import static org.mockito.Mockito.when;
-
 import static org.neo4j.collection.primitive.PrimitiveLongCollections.emptyIterator;
 import static org.neo4j.kernel.api.properties.Property.stringProperty;
 import static org.neo4j.kernel.impl.store.record.IndexRule.constraintIndexRule;
@@ -238,6 +237,12 @@ public int getIndexedCount( long nodeId, Object propertyValue )
                     return count;
                 }
 
+                @Override
+                public Set<Class> valueTypesInIndex()
+                {
+                    throw new UnsupportedOperationException();
+                }
+
                 @Override
                 public long sampleIndex( Register.DoubleLong.Out sampler )
                 {

community/consistency-check/src/test/java/org/neo4j/consistency/store/StoreAssertions.java

@@ -52,7 +52,7 @@ public static void assertConsistentStore( File dir ) throws ConsistencyCheckInco
                         dir.getAbsolutePath(),
                         configuration,
                         ProgressMonitorFactory.NONE,
-                        StringLogger.DEV_NULL
+                        StringLogger.SYSTEM_ERR
                 );
 
         assertTrue( result.isSuccessful() );

community/kernel/src/main/java/org/neo4j/kernel/NeoStoreDataSource.java

@@ -101,6 +101,8 @@
 import org.neo4j.kernel.impl.store.StoreId;
 import org.neo4j.kernel.impl.store.record.SchemaRule;
 import org.neo4j.kernel.impl.storemigration.StoreUpgrader;
+import org.neo4j.kernel.impl.storemigration.StoreVersionCheck;
+import org.neo4j.kernel.impl.storemigration.UpgradableDatabase;
 import org.neo4j.kernel.impl.transaction.TransactionHeaderInformationFactory;
 import org.neo4j.kernel.impl.transaction.TransactionMonitor;
 import org.neo4j.kernel.impl.transaction.log.LogFile;
@@ -552,7 +554,8 @@ public void transactionRecovered( long txId )
     // of the dependency tree, starting at the bottom
     private void upgradeStore( File store, StoreUpgrader storeMigrationProcess, SchemaIndexProvider indexProvider )
     {
-        storeMigrationProcess.addParticipant( indexProvider.storeMigrationParticipant() );
+        UpgradableDatabase upgradableDatabase = new UpgradableDatabase( new StoreVersionCheck( fs ) );
+        storeMigrationProcess.addParticipant( indexProvider.storeMigrationParticipant( fs, upgradableDatabase ) );
         storeMigrationProcess.migrateIfNeeded( store.getParentFile(), indexProvider, pageCache );
     }
 

community/kernel/src/main/java/org/neo4j/kernel/api/index/ArrayEncoder.java

@@ -19,17 +19,16 @@
  */
 package org.neo4j.kernel.api.index;
 
+import sun.misc.BASE64Encoder;
+
 import java.io.IOException;
 import java.io.OutputStream;
 import java.lang.reflect.Array;
 
-import sun.misc.BASE64Encoder;
-
 import org.neo4j.kernel.impl.util.Charsets;
 
 public class ArrayEncoder
 {
-    private static final byte[] LINE_ENDING = System.lineSeparator().getBytes();
     private static final BASE64Encoder base64Encoder = new BASE64Encoder()
     {
         @Override
@@ -38,12 +37,22 @@ protected void encodeBufferPrefix( OutputStream out ) throws IOException
             // don't initialize the non-thread-safe state
         }
 
+        @Override
+        protected void encodeBufferSuffix( OutputStream outputStream ) throws IOException
+        {
+            // nothing to do here
+        }
+
+        @Override
+        protected void encodeLinePrefix( OutputStream outputStream, int i ) throws IOException
+        {
+            // nothing to do here
+        }
+
         @Override
         protected void encodeLineSuffix( OutputStream out ) throws IOException
         {
-            // don't use the non-thread-safe state, instead do the same thing in a thread safe way,
-            // although, this still makes the encoded value platform dependant...
-            out.write( LINE_ENDING );
+            // don't use the non-thread-safe state, but do nothing
         }
     };
 

community/kernel/src/main/java/org/neo4j/kernel/api/index/IndexReader.java

@@ -19,6 +19,9 @@
  */
 package org.neo4j.kernel.api.index;
 
+import java.util.Collections;
+import java.util.Set;
+
 import org.neo4j.collection.primitive.PrimitiveLongCollections;
 import org.neo4j.collection.primitive.PrimitiveLongIterator;
 import org.neo4j.graphdb.Resource;
@@ -50,6 +53,12 @@ public int getIndexedCount( long nodeId, Object propertyValue )
             return 0;
         }
 
+        @Override
+        public Set<Class> valueTypesInIndex()
+        {
+            return Collections.emptySet();
+        }
+
         @Override
         public long sampleIndex( DoubleLong.Out result )
         {
@@ -68,6 +77,12 @@ public void close()
      */
     int getIndexedCount( long nodeId, Object propertyValue );
 
+    /**
+     *
+     * @return the set of value types present in the index
+     */
+    Set<Class> valueTypesInIndex();
+
     /**
      * Sample this index (on the current thread)
      * @param result contains the unique values and the sampled size
@@ -97,6 +112,12 @@ public int getIndexedCount( long nodeId, Object propertyValue )
             return delegate.getIndexedCount( nodeId, propertyValue );
         }
 
+        @Override
+        public Set<Class> valueTypesInIndex()
+        {
+            return delegate.valueTypesInIndex();
+        }
+
         @Override
         public long sampleIndex( DoubleLong.Out result ) throws IndexNotFoundKernelException
         {

community/kernel/src/main/java/org/neo4j/kernel/api/index/SchemaIndexProvider.java

@@ -28,10 +28,12 @@
 import org.neo4j.graphdb.DependencyResolver.SelectionStrategy;
 import org.neo4j.graphdb.ResourceIterator;
 import org.neo4j.helpers.collection.IteratorUtil;
+import org.neo4j.io.fs.FileSystemAbstraction;
 import org.neo4j.kernel.configuration.Config;
 import org.neo4j.kernel.impl.api.index.IndexingService;
 import org.neo4j.kernel.impl.api.index.sampling.IndexSamplingConfig;
 import org.neo4j.kernel.impl.storemigration.StoreMigrationParticipant;
+import org.neo4j.kernel.impl.storemigration.UpgradableDatabase;
 import org.neo4j.kernel.lifecycle.LifecycleAdapter;
 
 import static org.neo4j.graphdb.factory.GraphDatabaseSettings.store_dir;
@@ -96,41 +98,47 @@
  * {@link #getOnlineAccessor(long, IndexConfiguration, IndexSamplingConfig) online accessor} to
  * write to the index.
  */
-public abstract class SchemaIndexProvider extends LifecycleAdapter
-        implements Comparable<SchemaIndexProvider>
+public abstract class SchemaIndexProvider extends LifecycleAdapter implements Comparable<SchemaIndexProvider>
 {
     public static final SchemaIndexProvider NO_INDEX_PROVIDER =
-            new SchemaIndexProvider( new Descriptor("no-index-provider", "1.0"), -1 )
-    {
-        private final IndexAccessor singleWriter = new IndexAccessor.Adapter();
-        private final IndexPopulator singlePopulator = new IndexPopulator.Adapter();
-
-        @Override
-        public IndexAccessor getOnlineAccessor( long indexId, IndexConfiguration config,
-                                                IndexSamplingConfig samplingConfig )
-        {
-            return singleWriter;
-        }
-
-        @Override
-        public IndexPopulator getPopulator( long indexId, IndexDescriptor descriptor, IndexConfiguration config,
-                                            IndexSamplingConfig samplingConfig )
-        {
-            return singlePopulator;
-        }
-
-        @Override
-        public InternalIndexState getInitialState( long indexId )
-        {
-            return InternalIndexState.POPULATING;
-        }
-
-        @Override
-        public String getPopulationFailure( long indexId ) throws IllegalStateException
-        {
-            throw new IllegalStateException();
-        }
-    };
+            new SchemaIndexProvider( new Descriptor( "no-index-provider", "1.0" ), -1 )
+            {
+                private final IndexAccessor singleWriter = new IndexAccessor.Adapter();
+                private final IndexPopulator singlePopulator = new IndexPopulator.Adapter();
+
+                @Override
+                public IndexAccessor getOnlineAccessor( long indexId, IndexConfiguration config,
+                                                        IndexSamplingConfig samplingConfig )
+                {
+                    return singleWriter;
+                }
+
+                @Override
+                public IndexPopulator getPopulator( long indexId, IndexDescriptor descriptor, IndexConfiguration config,
+                                                    IndexSamplingConfig samplingConfig )
+                {
+                    return singlePopulator;
+                }
+
+                @Override
+                public InternalIndexState getInitialState( long indexId )
+                {
+                    return InternalIndexState.POPULATING;
+                }
+
+                @Override
+                public StoreMigrationParticipant storeMigrationParticipant( FileSystemAbstraction fs,
+                                                                            UpgradableDatabase upgradableDatabase )
+                {
+                    return StoreMigrationParticipant.NOT_PARTICIPATING;
+                }
+
+                @Override
+                public String getPopulationFailure( long indexId ) throws IllegalStateException
+                {
+                    throw new IllegalStateException();
+                }
+            };
 
     public static final SelectionStrategy HIGHEST_PRIORITIZED_OR_NONE =
             new SelectionStrategy()
@@ -238,10 +246,8 @@ public static File getRootDirectory( File storeDir, String key )
         return new File( new File( new File( storeDir, "schema" ), "index" ), key );
     }
 
-    public StoreMigrationParticipant storeMigrationParticipant()
-    {
-        return StoreMigrationParticipant.NOT_PARTICIPATING;
-    }
+    public abstract StoreMigrationParticipant storeMigrationParticipant( FileSystemAbstraction fs,
+                                                                         UpgradableDatabase upgradableDatabase );
 
     /**
      * Provides a snapshot of meta files about this index provider, not the indexes themselves.

community/kernel/src/main/java/org/neo4j/kernel/impl/api/index/sampling/IndexSamplingConfig.java

@@ -54,4 +54,34 @@ public boolean backgroundSampling()
     {
         return backgroundSampling;
     }
+
+    @Override
+    public boolean equals( Object o )
+    {
+        if ( this == o )
+        {
+            return true;
+        }
+
+        if ( o == null || getClass() != o.getClass() )
+        {
+            return false;
+        }
+
+        IndexSamplingConfig that = (IndexSamplingConfig) o;
+
+        return backgroundSampling == that.backgroundSampling &&
+               bufferSize == that.bufferSize &&
+               Double.compare( that.updateRatio, updateRatio ) == 0;
+    }
+
+    @Override
+    public int hashCode()
+    {
+        int result = bufferSize;
+        long temp = Double.doubleToLongBits( updateRatio );
+        result = 31 * result + (int) (temp ^ (temp >>> 32));
+        result = 31 * result + (backgroundSampling ? 1 : 0);
+        return result;
+    }
 }

community/kernel/src/main/java/org/neo4j/kernel/impl/store/CommonAbstractStore.java

@@ -58,7 +58,7 @@
  */
 public abstract class CommonAbstractStore implements IdSequence, AutoCloseable
 {
-    public static final String ALL_STORES_VERSION = "v0.A.4";
+    public static final String ALL_STORES_VERSION = "v0.A.5";
     public static final String UNKNOWN_VERSION = "Unknown";
     protected final Config configuration;
     protected final PageCache pageCache;