Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Ability to import > 2.4 billion nodes in import tool
Previously there was a limit of how many nodes could be imported in one import using the import tool. That limit came from the "tracker", the data stucture used for sorting ids after all had been added, only being capable of handling int-range values. This to save 4 bytes compared to using long for that always. As imports grow bigger this needs to change, so this commit introduces support for this tracker being long-range, but doesn't penalize < 2.4 billion imports because the type of data structure can be decided after all nodes have been imported and at that time it knows how many there are ans so the long-range data structure is only used for the imports exceeding that threshold.
- Loading branch information
Showing
15 changed files
with
580 additions
and
81 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
52 changes: 52 additions & 0 deletions
52
...c/main/java/org/neo4j/unsafe/impl/batchimport/cache/idmapping/string/AbstractTracker.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,52 @@ | ||
| /* | ||
| * Copyright (c) 2002-2016 "Neo Technology," | ||
| * Network Engine for Objects in Lund AB [http://neotechnology.com] | ||
| * | ||
| * This file is part of Neo4j. | ||
| * | ||
| * Neo4j is free software: you can redistribute it and/or modify | ||
| * it under the terms of the GNU General Public License as published by | ||
| * the Free Software Foundation, either version 3 of the License, or | ||
| * (at your option) any later version. | ||
| * | ||
| * This program is distributed in the hope that it will be useful, | ||
| * but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
| * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
| * GNU General Public License for more details. | ||
| * | ||
| * You should have received a copy of the GNU General Public License | ||
| * along with this program. If not, see <http://www.gnu.org/licenses/>. | ||
| */ | ||
| package org.neo4j.unsafe.impl.batchimport.cache.idmapping.string; | ||
|
|
||
| import org.neo4j.unsafe.impl.batchimport.cache.MemoryStatsVisitor; | ||
| import org.neo4j.unsafe.impl.batchimport.cache.NumberArray; | ||
|
|
||
| /** | ||
| * Base implementation of {@link Tracker} over a {@link NumberArray}. | ||
| * | ||
| * @param <ARRAY> type of {@link NumberArray} in this implementation. | ||
| */ | ||
| abstract class AbstractTracker<ARRAY extends NumberArray> implements Tracker | ||
| { | ||
| static final int DEFAULT_VALUE = -1; | ||
|
|
||
| protected ARRAY array; | ||
|
|
||
| protected AbstractTracker( ARRAY array ) | ||
| { | ||
| this.array = array; | ||
| } | ||
|
|
||
| @Override | ||
| public void acceptMemoryStatsVisitor( MemoryStatsVisitor visitor ) | ||
| { | ||
| array.acceptMemoryStatsVisitor( visitor ); | ||
| } | ||
|
|
||
| @Override | ||
| public void swap( long fromIndex, long toIndex, int count ) | ||
| { | ||
| array.swap( fromIndex, toIndex, count ); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
50 changes: 50 additions & 0 deletions
50
...el/src/main/java/org/neo4j/unsafe/impl/batchimport/cache/idmapping/string/IntTracker.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,50 @@ | ||
| /* | ||
| * Copyright (c) 2002-2016 "Neo Technology," | ||
| * Network Engine for Objects in Lund AB [http://neotechnology.com] | ||
| * | ||
| * This file is part of Neo4j. | ||
| * | ||
| * Neo4j is free software: you can redistribute it and/or modify | ||
| * it under the terms of the GNU General Public License as published by | ||
| * the Free Software Foundation, either version 3 of the License, or | ||
| * (at your option) any later version. | ||
| * | ||
| * This program is distributed in the hope that it will be useful, | ||
| * but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
| * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
| * GNU General Public License for more details. | ||
| * | ||
| * You should have received a copy of the GNU General Public License | ||
| * along with this program. If not, see <http://www.gnu.org/licenses/>. | ||
| */ | ||
| package org.neo4j.unsafe.impl.batchimport.cache.idmapping.string; | ||
|
|
||
| import org.neo4j.unsafe.impl.batchimport.Utils; | ||
| import org.neo4j.unsafe.impl.batchimport.cache.IntArray; | ||
|
|
||
| /** | ||
| * {@link Tracker} capable of keeping {@code int} range values, using {@link IntArray}. | ||
| * Will fail in {@link #set(long, long)} with {@link ArithmeticException} if trying to put a too big value. | ||
| */ | ||
| public class IntTracker extends AbstractTracker<IntArray> | ||
| { | ||
| public IntTracker( IntArray array ) | ||
| { | ||
| super( array ); | ||
| } | ||
|
|
||
| @Override | ||
| public long get( long index ) | ||
| { | ||
| return array.get( index ); | ||
| } | ||
|
|
||
| /** | ||
| * @throws ArithmeticException if value is bigger than {@link Integer#MAX_VALUE}. | ||
| */ | ||
| @Override | ||
| public void set( long index, long value ) | ||
| { | ||
| array.set( index, Utils.safeCastLongToInt( value ) ); | ||
| } | ||
| } |
45 changes: 45 additions & 0 deletions
45
...l/src/main/java/org/neo4j/unsafe/impl/batchimport/cache/idmapping/string/LongTracker.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,45 @@ | ||
| /* | ||
| * Copyright (c) 2002-2016 "Neo Technology," | ||
| * Network Engine for Objects in Lund AB [http://neotechnology.com] | ||
| * | ||
| * This file is part of Neo4j. | ||
| * | ||
| * Neo4j is free software: you can redistribute it and/or modify | ||
| * it under the terms of the GNU General Public License as published by | ||
| * the Free Software Foundation, either version 3 of the License, or | ||
| * (at your option) any later version. | ||
| * | ||
| * This program is distributed in the hope that it will be useful, | ||
| * but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
| * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
| * GNU General Public License for more details. | ||
| * | ||
| * You should have received a copy of the GNU General Public License | ||
| * along with this program. If not, see <http://www.gnu.org/licenses/>. | ||
| */ | ||
| package org.neo4j.unsafe.impl.batchimport.cache.idmapping.string; | ||
|
|
||
| import org.neo4j.unsafe.impl.batchimport.cache.LongArray; | ||
|
|
||
| /** | ||
| * {@link Tracker} capable of keeping {@code long} range values, using {@link LongArray}. | ||
| */ | ||
| public class LongTracker extends AbstractTracker<LongArray> | ||
| { | ||
| public LongTracker( LongArray array ) | ||
| { | ||
| super( array ); | ||
| } | ||
|
|
||
| @Override | ||
| public long get( long index ) | ||
| { | ||
| return array.get( index ); | ||
| } | ||
|
|
||
| @Override | ||
| public void set( long index, long value ) | ||
| { | ||
| array.set( index, value ); | ||
| } | ||
| } |
Oops, something went wrong.