Bumps store version of HighLimit format

there was a change in 3.0.5 which made a backwards compatible change to
HighLimit format. The store version wasn't bumped since there was no need
for actual migration. Not bumping store version leads to problems if
user, after running on 3.0.5 would start db on older 3.0.x patch.
The db would start normally since the store version would be correct,
but the data in the records would look completely different.

This commit fixes this problem and adds a temporary, 3.0-specific
special case as to not do actual migration when bumping store version.
3.1 has better mechanics so that this can be done w/o special-casing as well.

There will still be a problem with 3.0.5 in that going back from that version
to a lower 3.0.x patch release if using high limit record format will
fail on reading those records, or succeed but reading completely invalid
data in those records.

community/kernel/src/main/java/org/neo4j/kernel/impl/store/format/StoreVersion.java

@@ -32,7 +32,8 @@ public enum StoreVersion
     STANDARD_V2_3( "v0.A.6", true ),
     STANDARD_V3_0( "v0.A.7", true ),
 
-    HIGH_LIMIT_V3_0( "vE.H.0", false );
+    HIGH_LIMIT_V3_0_0( "vE.H.0", false ),
+    HIGH_LIMIT_V3_0_6( "vE.H.0b", false );
 
     private static final StoreVersion[] ALL_STORE_VERSIONS = values();
 

community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigrator.java

@@ -167,6 +167,27 @@ public void migrate( File storeDir, File migrationDir, MigrationProgressMonitor.
         writeLastTxInformation( migrationDir, lastTxInfo );
         writeLastTxLogPosition( migrationDir, lastTxLogPosition );
 
+        if ( versionToMigrateFrom.equals( "vE.H.0" ) )
+        {
+            // NOTE for 3.0 here is a special case for vE.H.0 "from" record format.
+            // Legend has it that 3.0.5 enterprise changed store format without changing store version.
+            // This was done to cheat the migrator to avoid doing store migration since the
+            // format itself was backwards compatible. Immediately a problem was detected:
+            // if a user uses 3.0.5 for a while and then goes back to a previous 3.0.x patch release
+            // the db wouldn't recognize it was an incompatible downgrade and start up normally,
+            // but read records with scrambled values and pointers, sort of.
+            //
+            // This condition has two functions:
+            //  1. preventing actual store migration between vE.H.0 --> vE.H.0b
+            //  2. making vE.H.0b used in any migration where either vE.H.0 or vE.H.0b is the existing format,
+            //     this because vE.H.0b is a superset of vE.H.0 and sometimes (for 3.0.5) vE.H.0
+            //     actually means vE.H.0b (in later version).
+            //
+            // In later versions of neo4j there are better mechanics in place so that a non-migration like this
+            // can be performed w/o special casing. To not require backporting that functionality
+            // this condition is here and should be removed in 3.1.
+            versionToMigrateFrom = "vE.H.0b";
+        }
         RecordFormats oldFormat = selectForVersion( versionToMigrateFrom );
         RecordFormats newFormat = selectForVersion( versionToMigrateTo );
         if ( !oldFormat.equals( newFormat ) )

community/kernel/src/test/java/org/neo4j/kernel/impl/store/format/StoreVersionTest.java

@@ -67,7 +67,7 @@ public static class EnterpriseVersions
         @Parameterized.Parameters( name = "{0}" )
         public static Collection<String> versions()
         {
-            return Arrays.asList( StoreVersion.HIGH_LIMIT_V3_0.versionString() );
+            return Arrays.asList( StoreVersion.HIGH_LIMIT_V3_0_0.versionString() );
         }
 
         @Test

community/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/UpgradableDatabaseTest.java

@@ -211,7 +211,7 @@ public static class UnsupportedVersions
         @Parameterized.Parameters( name = "{0}" )
         public static Collection<String> versions()
         {
-            return Arrays.asList( "v0.9.5", "v0.A.4", StoreVersion.HIGH_LIMIT_V3_0.versionString() );
+            return Arrays.asList( "v0.9.5", "v0.A.4", StoreVersion.HIGH_LIMIT_V3_0_0.versionString() );
         }
 
         @Rule

enterprise/kernel/src/main/java/org/neo4j/kernel/impl/store/format/highlimit/HighLimit.java

@@ -48,7 +48,7 @@ public class HighLimit extends BaseRecordFormats
      */
     static final int DEFAULT_MAXIMUM_BITS_PER_ID = 50;
 
-    public static final String STORE_VERSION = StoreVersion.HIGH_LIMIT_V3_0.versionString();
+    public static final String STORE_VERSION = StoreVersion.HIGH_LIMIT_V3_0_6.versionString();
     public static final RecordFormats RECORD_FORMATS = new HighLimit();
     public static final String NAME = "high_limit";
 

enterprise/kernel/src/main/java/org/neo4j/kernel/impl/store/format/highlimit/v30/BaseHighLimitRecordFormatV3_0.java → enterprise/kernel/src/main/java/org/neo4j/kernel/impl/store/format/highlimit/v300/BaseHighLimitRecordFormatV3_0_0.java

@@ -17,7 +17,7 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
-package org.neo4j.kernel.impl.store.format.highlimit.v30;
+package org.neo4j.kernel.impl.store.format.highlimit.v300;
 
 import java.io.IOException;
 import java.util.function.Function;
@@ -77,7 +77,7 @@
  *
  * @param <RECORD> type of {@link AbstractBaseRecord}
  */
-abstract class BaseHighLimitRecordFormatV3_0<RECORD extends AbstractBaseRecord>
+abstract class BaseHighLimitRecordFormatV3_0_0<RECORD extends AbstractBaseRecord>
         extends BaseOneByteHeaderRecordFormat<RECORD>
 {
     private static final int HEADER_BYTE = Byte.BYTES;
@@ -86,9 +86,9 @@ abstract class BaseHighLimitRecordFormatV3_0<RECORD extends AbstractBaseRecord>
     static final int HEADER_BIT_RECORD_UNIT = 0b0000_0010;
     static final int HEADER_BIT_FIRST_RECORD_UNIT = 0b0000_0100;
 
-    protected BaseHighLimitRecordFormatV3_0( Function<StoreHeader,Integer> recordSize, int recordHeaderSize )
+    protected BaseHighLimitRecordFormatV3_0_0( Function<StoreHeader,Integer> recordSize, int recordHeaderSize )
     {
-        super( recordSize, recordHeaderSize, IN_USE_BIT, HighLimitV3_0.DEFAULT_MAXIMUM_BITS_PER_ID );
+        super( recordSize, recordHeaderSize, IN_USE_BIT, HighLimitV3_0_0.DEFAULT_MAXIMUM_BITS_PER_ID );
     }
 
     public void read( RECORD record, PageCursor primaryCursor, RecordLoad mode, int recordSize )

enterprise/kernel/src/main/java/org/neo4j/kernel/impl/store/format/highlimit/v300/HighLimitFactoryV3_0_0.java

@@ -0,0 +1,38 @@
+/*
+ * Copyright (c) 2002-2016 "Neo Technology,"
+ * Network Engine for Objects in Lund AB [http://neotechnology.com]
+ *
+ * This file is part of Neo4j.
+ *
+ * Neo4j is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.neo4j.kernel.impl.store.format.highlimit.v300;
+
+import org.neo4j.helpers.Service;
+import org.neo4j.kernel.impl.store.format.RecordFormats;
+
+@Service.Implementation( RecordFormats.Factory.class )
+public class HighLimitFactoryV3_0_0 extends RecordFormats.Factory
+{
+    public HighLimitFactoryV3_0_0()
+    {
+        super( HighLimitV3_0_0.NAME );
+    }
+
+    @Override
+    public RecordFormats newInstance()
+    {
+        return HighLimitV3_0_0.RECORD_FORMATS;
+    }
+}

enterprise/kernel/src/main/java/org/neo4j/kernel/impl/store/format/highlimit/v30/HighLimitV3_0.java → enterprise/kernel/src/main/java/org/neo4j/kernel/impl/store/format/highlimit/v300/HighLimitV3_0_0.java

@@ -17,7 +17,7 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
-package org.neo4j.kernel.impl.store.format.highlimit.v30;
+package org.neo4j.kernel.impl.store.format.highlimit.v300;
 
 import org.neo4j.kernel.impl.store.format.BaseRecordFormats;
 import org.neo4j.kernel.impl.store.format.Capability;
@@ -40,46 +40,53 @@
 /**
  * Record format with very high limits, 50-bit per ID, while at the same time keeping store size small.
  *
- * @see BaseHighLimitRecordFormatV3_0
+ * NOTE: this format is also vE.H.0, but it's the first incarnation of it, without fixed references.
+ * The reason the same store version was kept when introducing fixed references was to avoid migration
+ * because the change was backwards compatible. Although this turned out to be a mistake because the
+ * format isn't forwards compatible and the way we prevent downgrading a db is by using store version,
+ * therefore we cannot prevent opening a db with fixed reference format on a neo4j patch version before
+ * fixed references were introduced (3.0.4).
+ *
+ * @see BaseHighLimitRecordFormatV3_0_0
  */
-public class HighLimitV3_0 extends BaseRecordFormats
+public class HighLimitV3_0_0 extends BaseRecordFormats
 {
     /**
      * Default maximum number of bits that can be used to represent id
      */
     static final int DEFAULT_MAXIMUM_BITS_PER_ID = 50;
 
-    public static final String STORE_VERSION = StoreVersion.HIGH_LIMIT_V3_0.versionString();
-    public static final RecordFormats RECORD_FORMATS = new HighLimitV3_0();
-    public static final String NAME = "high_limitV3_0";
+    public static final String STORE_VERSION = StoreVersion.HIGH_LIMIT_V3_0_0.versionString();
+    public static final RecordFormats RECORD_FORMATS = new HighLimitV3_0_0();
+    public static final String NAME = "high_limitV3_0_0";
 
-    public HighLimitV3_0()
+    public HighLimitV3_0_0()
     {
         super( STORE_VERSION, 7, Capability.DENSE_NODES, Capability.SCHEMA, Capability.LUCENE_5 );
     }
 
     @Override
     public RecordFormat<NodeRecord> node()
     {
-        return new NodeRecordFormatV3_0();
+        return new NodeRecordFormatV3_0_0();
     }
 
     @Override
     public RecordFormat<RelationshipRecord> relationship()
     {
-        return new RelationshipRecordFormatV3_0();
+        return new RelationshipRecordFormatV3_0_0();
     }
 
     @Override
     public RecordFormat<RelationshipGroupRecord> relationshipGroup()
     {
-        return new RelationshipGroupRecordFormatV3_0();
+        return new RelationshipGroupRecordFormatV3_0_0();
     }
 
     @Override
     public RecordFormat<PropertyRecord> property()
     {
-        return new PropertyRecordFormatV3_0();
+        return new PropertyRecordFormatV3_0_0();
     }
 
     @Override

enterprise/kernel/src/main/java/org/neo4j/kernel/impl/store/format/highlimit/v30/NodeRecordFormatV3_0.java → enterprise/kernel/src/main/java/org/neo4j/kernel/impl/store/format/highlimit/v300/NodeRecordFormatV3_0_0.java

@@ -17,7 +17,7 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
-package org.neo4j.kernel.impl.store.format.highlimit.v30;
+package org.neo4j.kernel.impl.store.format.highlimit.v300;
 
 import java.io.IOException;
 
@@ -37,7 +37,7 @@
  *
  * => 12B-22B
  */
-public class NodeRecordFormatV3_0 extends BaseHighLimitRecordFormatV3_0<NodeRecord>
+public class NodeRecordFormatV3_0_0 extends BaseHighLimitRecordFormatV3_0_0<NodeRecord>
 {
     public static final int RECORD_SIZE = 16;
 
@@ -47,12 +47,12 @@ public class NodeRecordFormatV3_0 extends BaseHighLimitRecordFormatV3_0<NodeReco
     private static final int HAS_PROPERTY_BIT     = 0b0010_0000;
     private static final int HAS_LABELS_BIT       = 0b0100_0000;
 
-    public NodeRecordFormatV3_0()
+    public NodeRecordFormatV3_0_0()
     {
         this( RECORD_SIZE );
     }
 
-    NodeRecordFormatV3_0( int recordSize )
+    NodeRecordFormatV3_0_0( int recordSize )
     {
         super( fixedRecordSize( recordSize ), 0 );
     }

enterprise/kernel/src/main/java/org/neo4j/kernel/impl/store/format/highlimit/v30/PropertyRecordFormatV3_0.java → enterprise/kernel/src/main/java/org/neo4j/kernel/impl/store/format/highlimit/v300/PropertyRecordFormatV3_0_0.java

@@ -17,7 +17,7 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
-package org.neo4j.kernel.impl.store.format.highlimit.v30;
+package org.neo4j.kernel.impl.store.format.highlimit.v300;
 
 import java.io.IOException;
 
@@ -47,13 +47,13 @@
  *
  * => 39B-49B
  */
-public class PropertyRecordFormatV3_0 extends BaseOneByteHeaderRecordFormat<PropertyRecord>
+public class PropertyRecordFormatV3_0_0 extends BaseOneByteHeaderRecordFormat<PropertyRecord>
 {
     public static final int RECORD_SIZE = 48;
 
-    public PropertyRecordFormatV3_0()
+    public PropertyRecordFormatV3_0_0()
     {
-        super( fixedRecordSize( RECORD_SIZE ), 0, IN_USE_BIT, HighLimitV3_0.DEFAULT_MAXIMUM_BITS_PER_ID );
+        super( fixedRecordSize( RECORD_SIZE ), 0, IN_USE_BIT, HighLimitV3_0_0.DEFAULT_MAXIMUM_BITS_PER_ID );
     }
 
     @Override

enterprise/kernel/src/main/java/org/neo4j/kernel/impl/store/format/highlimit/v30/RelationshipGroupRecordFormatV3_0.java → enterprise/kernel/src/main/java/org/neo4j/kernel/impl/store/format/highlimit/v300/RelationshipGroupRecordFormatV3_0_0.java

@@ -17,7 +17,7 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
-package org.neo4j.kernel.impl.store.format.highlimit.v30;
+package org.neo4j.kernel.impl.store.format.highlimit.v300;
 
 import java.io.IOException;
 
@@ -39,7 +39,7 @@
  *
  * => 18B-43B
  */
-public class RelationshipGroupRecordFormatV3_0 extends BaseHighLimitRecordFormatV3_0<RelationshipGroupRecord>
+public class RelationshipGroupRecordFormatV3_0_0 extends BaseHighLimitRecordFormatV3_0_0<RelationshipGroupRecord>
 {
     public static final int RECORD_SIZE = 32;
 
@@ -48,12 +48,12 @@ public class RelationshipGroupRecordFormatV3_0 extends BaseHighLimitRecordFormat
     private static final int HAS_LOOP_BIT     = 0b0010_0000;
     private static final int HAS_NEXT_BIT     = 0b0100_0000;
 
-    public RelationshipGroupRecordFormatV3_0()
+    public RelationshipGroupRecordFormatV3_0_0()
     {
         this( RECORD_SIZE );
     }
 
-    RelationshipGroupRecordFormatV3_0( int recordSize )
+    RelationshipGroupRecordFormatV3_0_0( int recordSize )
     {
         super( fixedRecordSize( recordSize ), 0 );
     }

enterprise/kernel/src/main/java/org/neo4j/kernel/impl/store/format/highlimit/v30/RelationshipRecordFormatV3_0.java → enterprise/kernel/src/main/java/org/neo4j/kernel/impl/store/format/highlimit/v300/RelationshipRecordFormatV3_0_0.java

@@ -17,7 +17,7 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
-package org.neo4j.kernel.impl.store.format.highlimit.v30;
+package org.neo4j.kernel.impl.store.format.highlimit.v300;
 
 import java.io.IOException;
 
@@ -44,7 +44,7 @@
  *
  * => 24B-59B
  */
-public class RelationshipRecordFormatV3_0 extends BaseHighLimitRecordFormatV3_0<RelationshipRecord>
+public class RelationshipRecordFormatV3_0_0 extends BaseHighLimitRecordFormatV3_0_0<RelationshipRecord>
 {
     public static final int RECORD_SIZE = 32;
 
@@ -54,12 +54,12 @@ public class RelationshipRecordFormatV3_0 extends BaseHighLimitRecordFormatV3_0<
     private static final int HAS_SECOND_CHAIN_NEXT_BIT = 0b0100_0000;
     private static final int HAS_PROPERTY_BIT = 0b1000_0000;
 
-    public RelationshipRecordFormatV3_0()
+    public RelationshipRecordFormatV3_0_0()
     {
         this( RECORD_SIZE );
     }
 
-    RelationshipRecordFormatV3_0( int recordSize )
+    RelationshipRecordFormatV3_0_0( int recordSize )
     {
         super( fixedRecordSize( recordSize ), 0 );
     }

enterprise/kernel/src/main/resources/META-INF/services/org.neo4j.kernel.impl.store.format.RecordFormats$Factory

@@ -1 +1,2 @@
-org.neo4j.kernel.impl.store.format.highlimit.HighLimitFactory
+org.neo4j.kernel.impl.store.format.highlimit.HighLimitFactory
+org.neo4j.kernel.impl.store.format.highlimit.v300.HighLimitFactoryV3_0_0

enterprise/kernel/src/test/java/org/neo4j/kernel/impl/store/format/highlimit/NodeRecordFormatTest.java

@@ -28,7 +28,7 @@
 import java.util.concurrent.ThreadLocalRandom;
 
 import org.neo4j.io.ByteUnit;
-import org.neo4j.kernel.impl.store.format.highlimit.v30.NodeRecordFormatV3_0;
+import org.neo4j.kernel.impl.store.format.highlimit.v300.NodeRecordFormatV3_0_0;
 import org.neo4j.kernel.impl.store.record.NodeRecord;
 import org.neo4j.kernel.impl.store.record.Record;
 import org.neo4j.kernel.impl.store.record.RecordLoad;
@@ -184,8 +184,8 @@ public void readDoubleUnitRecordStoredNotInFixedReferenceFormat() throws Excepti
 
     private void writeRecordWithOldFormat( NodeRecord oldFormatRecord ) throws IOException
     {
-        int oldRecordSize = NodeRecordFormatV3_0.RECORD_SIZE;
-        NodeRecordFormatV3_0 recordFormatV30 = new NodeRecordFormatV3_0();
+        int oldRecordSize = NodeRecordFormatV3_0_0.RECORD_SIZE;
+        NodeRecordFormatV3_0_0 recordFormatV30 = new NodeRecordFormatV3_0_0();
         recordFormatV30.prepare( oldFormatRecord, oldRecordSize, idSequence );
         recordFormatV30.write( oldFormatRecord, pageCursor, oldRecordSize );
         pageCursor.setOffset( 0 );

enterprise/kernel/src/test/java/org/neo4j/kernel/impl/store/format/highlimit/PropertyRecordFormatTest.java

@@ -29,7 +29,7 @@
 import org.neo4j.io.ByteUnit;
 import org.neo4j.io.pagecache.StubPageCursor;
 import org.neo4j.kernel.impl.store.IntStoreHeader;
-import org.neo4j.kernel.impl.store.format.highlimit.v30.PropertyRecordFormatV3_0;
+import org.neo4j.kernel.impl.store.format.highlimit.v300.PropertyRecordFormatV3_0_0;
 import org.neo4j.kernel.impl.store.record.PropertyRecord;
 import org.neo4j.kernel.impl.store.record.Record;
 import org.neo4j.kernel.impl.store.record.RecordLoad;
@@ -199,8 +199,8 @@ public void readSingleUnitRecordStoredNotInFixedReferenceFormat() throws Excepti
 
     private void writeRecordWithOldFormat( PropertyRecord oldFormatRecord ) throws IOException
     {
-        int oldRecordSize = PropertyRecordFormatV3_0.RECORD_SIZE;
-        PropertyRecordFormatV3_0 recordFormatV30 = new PropertyRecordFormatV3_0();
+        int oldRecordSize = PropertyRecordFormatV3_0_0.RECORD_SIZE;
+        PropertyRecordFormatV3_0_0 recordFormatV30 = new PropertyRecordFormatV3_0_0();
         recordFormatV30.prepare( oldFormatRecord, oldRecordSize, idSequence );
         recordFormatV30.write( oldFormatRecord, pageCursor, oldRecordSize );
         pageCursor.setOffset( 0 );

enterprise/kernel/src/test/java/org/neo4j/kernel/impl/store/format/highlimit/RelationshipGroupRecordFormatTest.java

@@ -31,7 +31,7 @@
 import java.util.concurrent.ThreadLocalRandom;
 
 import org.neo4j.io.ByteUnit;
-import org.neo4j.kernel.impl.store.format.highlimit.v30.RelationshipGroupRecordFormatV3_0;
+import org.neo4j.kernel.impl.store.format.highlimit.v300.RelationshipGroupRecordFormatV3_0_0;
 import org.neo4j.kernel.impl.store.record.RecordLoad;
 import org.neo4j.kernel.impl.store.record.RelationshipGroupRecord;
 
@@ -157,8 +157,8 @@ public void readDoubleUnitRecordStoredNotInFixedReferenceFormat() throws Excepti
 
     private void writeRecordWithOldFormat( RelationshipGroupRecord oldFormatRecord ) throws IOException
     {
-        int oldRecordSize = RelationshipGroupRecordFormatV3_0.RECORD_SIZE;
-        RelationshipGroupRecordFormatV3_0 recordFormatV30 = new RelationshipGroupRecordFormatV3_0();
+        int oldRecordSize = RelationshipGroupRecordFormatV3_0_0.RECORD_SIZE;
+        RelationshipGroupRecordFormatV3_0_0 recordFormatV30 = new RelationshipGroupRecordFormatV3_0_0();
         recordFormatV30.prepare( oldFormatRecord, oldRecordSize, idSequence );
         recordFormatV30.write( oldFormatRecord, pageCursor, oldRecordSize );
         pageCursor.setOffset( 0 );