Allow to load and use 3.0 high limit format (without fixed references…

… part) as independent HighLimitV3_0 record format.

Update high limit format store version to `vE.H.1`.

community/kernel/src/main/java/org/neo4j/kernel/impl/store/format/Capability.java

@@ -37,6 +37,11 @@ public enum Capability
      */
     DENSE_NODES( CapabilityType.FORMAT, CapabilityType.STORE ),
 
+    /**
+     * Store has fixed reference encoding support support
+     */
+    FIXED_REFERENCE( CapabilityType.FORMAT ),
+
     /**
      * Store has version trailers in the end of cleanly shut down store
      */

community/kernel/src/main/java/org/neo4j/kernel/impl/store/format/StoreVersion.java

@@ -32,7 +32,8 @@ public enum StoreVersion
     STANDARD_V2_3( "v0.A.6", true ),
     STANDARD_V3_0( "v0.A.7", true ),
 
-    HIGH_LIMIT_V3_0( "vE.H.0", false );
+    HIGH_LIMIT_V3_0( "vE.H.0", false ),
+    HIGH_LIMIT_V3_1( "vE.H.1", false );
 
     private static final StoreVersion[] ALL_STORE_VERSIONS = values();
 

community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigrator.java

@@ -55,6 +55,7 @@
 import org.neo4j.kernel.impl.store.StoreType;
 import org.neo4j.kernel.impl.store.TransactionId;
 import org.neo4j.kernel.impl.store.counts.CountsTracker;
+import org.neo4j.kernel.impl.store.format.CapabilityType;
 import org.neo4j.kernel.impl.store.format.RecordFormats;
 import org.neo4j.kernel.impl.store.format.standard.MetaDataRecordFormat;
 import org.neo4j.kernel.impl.store.format.standard.NodeRecordFormat;
@@ -169,7 +170,7 @@ public void migrate( File storeDir, File migrationDir, MigrationProgressMonitor.
 
         RecordFormats oldFormat = selectForVersion( versionToMigrateFrom );
         RecordFormats newFormat = selectForVersion( versionToMigrateTo );
-        if ( !oldFormat.equals( newFormat ) )
+        if ( !oldFormat.hasSameCapabilities( newFormat, CapabilityType.FORMAT ) )
         {
             // TODO if this store has relationship indexes then warn user about that they will be incorrect
             // after migration, because now we're rewriting the relationship ids.

community/neo4j/src/test/java/synchronization/ConcurrentChangesOnEntitiesTest.java

@@ -22,6 +22,7 @@
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.RuleChain;
 
 import java.io.IOException;
 import java.util.concurrent.CyclicBarrier;
@@ -41,15 +42,21 @@
 import org.neo4j.kernel.configuration.Config;
 import org.neo4j.logging.FormattedLogProvider;
 import org.neo4j.logging.LogProvider;
+import org.neo4j.test.rule.SuppressOutput;
 import org.neo4j.test.rule.TestDirectory;
 
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
 public class ConcurrentChangesOnEntitiesTest
 {
+
+    private SuppressOutput suppressOutput = SuppressOutput.suppressAll();
+    private TestDirectory testDirectory = TestDirectory.testDirectory();
+
     @Rule
-    public TestDirectory testDirectory = TestDirectory.testDirectory();
+    public RuleChain ruleChain = RuleChain.outerRule( suppressOutput ).around( testDirectory );
+
     private final CyclicBarrier barrier = new CyclicBarrier( 2 );
     private final AtomicReference<Exception> ex = new AtomicReference<>();
     private GraphDatabaseService db;

community/neo4j/src/test/java/upgrade/StoreUpgraderTest.java

@@ -32,6 +32,9 @@
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import org.neo4j.consistency.checking.full.ConsistencyCheckIncompleteException;
 import org.neo4j.graphdb.GraphDatabaseService;
@@ -85,6 +88,7 @@
 import static org.junit.Assert.assertThat;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
+import static org.junit.Assume.assumeTrue;
 import static org.mockito.Matchers.any;
 import static org.mockito.Matchers.eq;
 import static org.mockito.Mockito.verify;
@@ -139,7 +143,7 @@ public void prepareDb() throws IOException
     {
         dbDirectory = directory.directory( "db_" + version );
         File prepareDirectory = directory.directory( "prepare_" + version );
-        prepareSampleLegacyDatabase( version, fileSystem, dbDirectory, prepareDirectory );
+        prepareSampleDatabase( version, fileSystem, dbDirectory, prepareDirectory );
     }
 
     @Test
@@ -150,7 +154,11 @@ public void shouldUpgradeAnOldFormatStore() throws IOException, ConsistencyCheck
         UpgradableDatabase upgradableDatabase = new UpgradableDatabase( fileSystem,
                 new StoreVersionCheck( pageCache ), new LegacyStoreVersionCheck( fileSystem ),
                 getRecordFormats() );
-        assertEquals( !StandardV2_3.STORE_VERSION.equals( version ),
+        Set<String> versionSet = versionSet( StandardV2_0.STORE_VERSION,
+                                                       StandardV2_1.STORE_VERSION,
+                                                       StandardV2_2.STORE_VERSION );
+
+        assertEquals( versionSet.contains( version ),
                 allLegacyStoreFilesHaveVersion( fileSystem, dbDirectory, version ) );
 
         // When
@@ -194,7 +202,9 @@ public void shouldHaltUpgradeIfUpgradeConfigurationVetoesTheProcess()
     public void shouldLeaveAllFilesUntouchedIfWrongVersionNumberFound()
             throws IOException
     {
-        Assume.assumeFalse( StandardV2_3.STORE_VERSION.equals( version ) );
+        Set<String> applicableVersions =
+                versionSet( StandardV2_0.STORE_VERSION, StandardV2_1.STORE_VERSION, StandardV2_2.STORE_VERSION );
+        assumeTrue( "Applicable only to specified version set.", applicableVersions.contains( version ) );
 
         File comparisonDirectory = new File( "target/" + StoreUpgraderTest.class.getSimpleName()
                                              + "shouldLeaveAllFilesUntouchedIfWrongVersionNumberFound-comparison" );
@@ -221,11 +231,11 @@ public void shouldLeaveAllFilesUntouchedIfWrongVersionNumberFound()
     }
 
     @Test
-    public void shouldRefuseToUpgradeIfAnyOfTheStoresWeNotShutDownCleanly()
+    public void shouldRefuseToUpgradeIfAnyOfTheStoresWereNotShutDownCleanly()
             throws IOException
     {
-        File comparisonDirectory = new File( "target/" + StoreUpgraderTest.class.getSimpleName()
-                                             + "shouldRefuseToUpgradeIfAnyOfTheStoresWeNotShutDownCleanly-comparison" );
+        File comparisonDirectory = new File( "target/" + StoreUpgraderTest.class.getSimpleName() +
+                                             "shouldRefuseToUpgradeIfAnyOfTheStoresWereNotShutDownCleanly-comparison" );
         makeDbNotCleanlyShutdown( false );
         fileSystem.deleteRecursively( comparisonDirectory );
         fileSystem.copyRecursively( dbDirectory, comparisonDirectory );
@@ -251,8 +261,8 @@ public void shouldRefuseToUpgradeIfAnyOfTheStoresWeNotShutDownCleanly()
     public void shouldRefuseToUpgradeIfAllOfTheStoresWereNotShutDownCleanly()
             throws IOException
     {
-        File comparisonDirectory = new File( "target/" + StoreUpgraderTest.class.getSimpleName()
-                                             + "shouldRefuseToUpgradeIfAllOfTheStoresWeNotShutDownCleanly-comparison" );
+        File comparisonDirectory = new File( "target/" + StoreUpgraderTest.class.getSimpleName() +
+                                            "shouldRefuseToUpgradeIfAllOfTheStoresWereNotShutDownCleanly-comparison" );
         makeDbNotCleanlyShutdown( true );
         fileSystem.deleteRecursively( comparisonDirectory );
         fileSystem.copyRecursively( dbDirectory, comparisonDirectory );
@@ -390,6 +400,12 @@ public void upgraderShouldCleanupLegacyLeftoverAndMigrationDirs() throws Excepti
         assertThat( migrationHelperDirs(), is( emptyCollectionOf( File.class ) ) );
     }
 
+    protected void prepareSampleDatabase( String version, FileSystemAbstraction fileSystem, File dbDirectory,
+            File databaseDirectory ) throws IOException
+    {
+        prepareSampleLegacyDatabase( version, fileSystem, dbDirectory, databaseDirectory );
+    }
+
     private static void assertCorrectStoreVersion( String expectedStoreVersion, StoreVersionCheck check, File storeDir )
     {
         File neoStoreFile = new File( storeDir, MetaDataStore.DEFAULT_NAME );
@@ -449,11 +465,9 @@ private List<File> migrationHelperDirs()
 
     private void makeDbNotCleanlyShutdown( boolean truncateAll ) throws IOException
     {
-        if ( StandardV2_3.STORE_VERSION.equals( version ) )
-        {
-            removeCheckPointFromTxLog( fileSystem, dbDirectory );
-        }
-        else
+        Set<String> truncateVersions = versionSet( StandardV2_0.STORE_VERSION, StandardV2_1.STORE_VERSION,
+                StandardV2_2.STORE_VERSION );
+        if (truncateVersions.contains( version ))
         {
             if ( truncateAll )
             {
@@ -466,6 +480,10 @@ private void makeDbNotCleanlyShutdown( boolean truncateAll ) throws IOException
                 truncateFile( fileSystem, storeFile, "StringPropertyStore " + version );
             }
         }
+        else
+        {
+            removeCheckPointFromTxLog( fileSystem, dbDirectory );
+        }
     }
 
     private Config getTuningConfig()
@@ -483,6 +501,11 @@ protected String getRecordFormatsName()
         return StandardV3_0.NAME;
     }
 
+    private Set<String> versionSet(String... versions)
+    {
+        return Stream.of( versions ).collect( Collectors.toSet() );
+    }
+
     private void startStopDatabase()
     {
         GraphDatabaseService databaseService = new TestGraphDatabaseFactory().newEmbeddedDatabase( dbDirectory );

enterprise/kernel/src/main/java/org/neo4j/kernel/impl/store/format/highlimit/HighLimit.java

@@ -24,6 +24,7 @@
 import org.neo4j.kernel.impl.store.format.RecordFormat;
 import org.neo4j.kernel.impl.store.format.RecordFormats;
 import org.neo4j.kernel.impl.store.format.StoreVersion;
+import org.neo4j.kernel.impl.store.format.highlimit.v30.HighLimitV3_0;
 import org.neo4j.kernel.impl.store.format.standard.LabelTokenRecordFormat;
 import org.neo4j.kernel.impl.store.format.standard.PropertyKeyTokenRecordFormat;
 import org.neo4j.kernel.impl.store.format.standard.RelationshipTypeTokenRecordFormat;
@@ -48,13 +49,14 @@ public class HighLimit extends BaseRecordFormats
      */
     static final int DEFAULT_MAXIMUM_BITS_PER_ID = 50;
 
-    public static final String STORE_VERSION = StoreVersion.HIGH_LIMIT_V3_0.versionString();
+    public static final String STORE_VERSION = StoreVersion.HIGH_LIMIT_V3_1.versionString();
     public static final RecordFormats RECORD_FORMATS = new HighLimit();
     public static final String NAME = "high_limit";
 
     public HighLimit()
     {
-        super( STORE_VERSION, 8, Capability.DENSE_NODES, Capability.SCHEMA, Capability.LUCENE_5 );
+        super( STORE_VERSION, 8, Capability.DENSE_NODES, Capability.SCHEMA, Capability.LUCENE_5,
+                Capability.FIXED_REFERENCE );
     }
 
     @Override

enterprise/kernel/src/main/java/org/neo4j/kernel/impl/store/format/highlimit/v30/HighLimitV3_0.java

@@ -55,7 +55,8 @@ public class HighLimitV3_0 extends BaseRecordFormats
 
     public HighLimitV3_0()
     {
-        super( STORE_VERSION, 7, Capability.DENSE_NODES, Capability.SCHEMA, Capability.LUCENE_5 );
+        super( STORE_VERSION, 7, Capability.DENSE_NODES, Capability.SCHEMA, Capability.LUCENE_5,
+                Capability.FIXED_REFERENCE );
     }
 
     @Override

enterprise/kernel/src/main/java/org/neo4j/kernel/impl/store/format/highlimit/v30/HighLimitV3_0Factory.java

@@ -0,0 +1,38 @@
+/*
+ * Copyright (c) 2002-2016 "Neo Technology,"
+ * Network Engine for Objects in Lund AB [http://neotechnology.com]
+ *
+ * This file is part of Neo4j.
+ *
+ * Neo4j is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.neo4j.kernel.impl.store.format.highlimit.v30;
+
+import org.neo4j.helpers.Service;
+import org.neo4j.kernel.impl.store.format.RecordFormats;
+
+@Service.Implementation( RecordFormats.Factory.class )
+public class HighLimitV3_0Factory extends RecordFormats.Factory
+{
+    public HighLimitV3_0Factory()
+    {
+        super( HighLimitV3_0.NAME );
+    }
+
+    @Override
+    public RecordFormats newInstance()
+    {
+        return HighLimitV3_0.RECORD_FORMATS;
+    }
+}

enterprise/kernel/src/main/resources/META-INF/services/org.neo4j.kernel.impl.store.format.RecordFormats$Factory

@@ -1 +1,2 @@
-org.neo4j.kernel.impl.store.format.highlimit.HighLimitFactory
+org.neo4j.kernel.impl.store.format.highlimit.HighLimitFactory
+org.neo4j.kernel.impl.store.format.highlimit.v30.HighLimitV3_0Factory

enterprise/kernel/src/test/java/org/neo4j/kernel/impl/store/format/RecordFormatSelectorTest.java

@@ -32,6 +32,7 @@
 import org.neo4j.kernel.configuration.Config;
 import org.neo4j.kernel.impl.store.MetaDataStore;
 import org.neo4j.kernel.impl.store.format.highlimit.HighLimit;
+import org.neo4j.kernel.impl.store.format.highlimit.v30.HighLimitV3_0;
 import org.neo4j.kernel.impl.store.format.standard.StandardV2_0;
 import org.neo4j.kernel.impl.store.format.standard.StandardV2_1;
 import org.neo4j.kernel.impl.store.format.standard.StandardV2_2;
@@ -83,6 +84,7 @@ public void selectForVersionTest()
         assertSame( StandardV2_2.RECORD_FORMATS, selectForVersion( StandardV2_2.STORE_VERSION ) );
         assertSame( StandardV2_3.RECORD_FORMATS, selectForVersion( StandardV2_3.STORE_VERSION ) );
         assertSame( StandardV3_0.RECORD_FORMATS, selectForVersion( StandardV3_0.STORE_VERSION ) );
+        assertSame( HighLimitV3_0.RECORD_FORMATS, selectForVersion( HighLimitV3_0.STORE_VERSION ) );
         assertSame( HighLimit.RECORD_FORMATS, selectForVersion( HighLimit.STORE_VERSION ) );
     }
 
@@ -136,6 +138,7 @@ public void selectForStoreWithValidStore() throws IOException
         verifySelectForStore( pageCache, StandardV2_2.RECORD_FORMATS );
         verifySelectForStore( pageCache, StandardV2_3.RECORD_FORMATS );
         verifySelectForStore( pageCache, StandardV3_0.RECORD_FORMATS );
+        verifySelectForStore( pageCache, HighLimitV3_0.RECORD_FORMATS );
         verifySelectForStore( pageCache, HighLimit.RECORD_FORMATS );
     }