Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Defragments relationship groups as part of import
now that the importer has been changed to import relationships per type it means that relationship groups will be more scattered, which will have read queries take a hit compared to previously. This commit introduces a relationship group defragmenter which is written using normal batch importer stages and steps and runs as part of import, after the relationships have been imported.
- Loading branch information
Showing
30 changed files
with
1,801 additions
and
57 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
49 changes: 49 additions & 0 deletions
49
community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/CacheGroupsStep.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,49 @@ | ||
| /* | ||
| * Copyright (c) 2002-2016 "Neo Technology," | ||
| * Network Engine for Objects in Lund AB [http://neotechnology.com] | ||
| * | ||
| * This file is part of Neo4j. | ||
| * | ||
| * Neo4j is free software: you can redistribute it and/or modify | ||
| * it under the terms of the GNU General Public License as published by | ||
| * the Free Software Foundation, either version 3 of the License, or | ||
| * (at your option) any later version. | ||
| * | ||
| * This program is distributed in the hope that it will be useful, | ||
| * but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
| * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
| * GNU General Public License for more details. | ||
| * | ||
| * You should have received a copy of the GNU General Public License | ||
| * along with this program. If not, see <http://www.gnu.org/licenses/>. | ||
| */ | ||
| package org.neo4j.unsafe.impl.batchimport; | ||
|
|
||
| import org.neo4j.kernel.impl.store.record.RelationshipGroupRecord; | ||
| import org.neo4j.unsafe.impl.batchimport.staging.BatchSender; | ||
| import org.neo4j.unsafe.impl.batchimport.staging.Configuration; | ||
| import org.neo4j.unsafe.impl.batchimport.staging.ProcessorStep; | ||
| import org.neo4j.unsafe.impl.batchimport.staging.StageControl; | ||
|
|
||
| /** | ||
| * Caches {@link RelationshipGroupRecord} into {@link RelationshipGroupCache}. | ||
| */ | ||
| public class CacheGroupsStep extends ProcessorStep<RelationshipGroupRecord[]> | ||
| { | ||
| private final RelationshipGroupCache cache; | ||
|
|
||
| public CacheGroupsStep( StageControl control, Configuration config, RelationshipGroupCache cache ) | ||
| { | ||
| super( control, "CACHE", config, 1 ); | ||
| this.cache = cache; | ||
| } | ||
|
|
||
| @Override | ||
| protected void process( RelationshipGroupRecord[] batch, BatchSender sender ) throws Throwable | ||
| { | ||
| for ( RelationshipGroupRecord groupRecord : batch ) | ||
| { | ||
| cache.put( groupRecord ); | ||
| } | ||
| } | ||
| } |
42 changes: 42 additions & 0 deletions
42
community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/CountGroupsStage.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,42 @@ | ||
| /* | ||
| * Copyright (c) 2002-2016 "Neo Technology," | ||
| * Network Engine for Objects in Lund AB [http://neotechnology.com] | ||
| * | ||
| * This file is part of Neo4j. | ||
| * | ||
| * Neo4j is free software: you can redistribute it and/or modify | ||
| * it under the terms of the GNU General Public License as published by | ||
| * the Free Software Foundation, either version 3 of the License, or | ||
| * (at your option) any later version. | ||
| * | ||
| * This program is distributed in the hope that it will be useful, | ||
| * but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
| * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
| * GNU General Public License for more details. | ||
| * | ||
| * You should have received a copy of the GNU General Public License | ||
| * along with this program. If not, see <http://www.gnu.org/licenses/>. | ||
| */ | ||
| package org.neo4j.unsafe.impl.batchimport; | ||
|
|
||
| import org.neo4j.kernel.impl.store.RecordStore; | ||
| import org.neo4j.kernel.impl.store.record.RelationshipGroupRecord; | ||
| import org.neo4j.unsafe.impl.batchimport.staging.Configuration; | ||
| import org.neo4j.unsafe.impl.batchimport.staging.ReadRecordsStep; | ||
| import org.neo4j.unsafe.impl.batchimport.staging.Stage; | ||
| import org.neo4j.unsafe.impl.batchimport.store.BatchingNeoStores; | ||
|
|
||
| /** | ||
| * Stage for counting groups per node, populates {@link RelationshipGroupCache}. | ||
| */ | ||
| public class CountGroupsStage extends Stage | ||
| { | ||
| public CountGroupsStage( Configuration config, BatchingNeoStores neoStore, RelationshipGroupCache groupCache ) | ||
| { | ||
| super( "Count groups", config ); | ||
|
|
||
| RecordStore<RelationshipGroupRecord> store = neoStore.getRelationshipGroupStore(); | ||
| add( new ReadRecordsStep<>( control(), config, store, RecordIdIteration.allIn( store ) ) ); | ||
| add( new CountGroupsStep( control(), config, groupCache ) ); | ||
| } | ||
| } |
50 changes: 50 additions & 0 deletions
50
community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/CountGroupsStep.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,50 @@ | ||
| /* | ||
| * Copyright (c) 2002-2016 "Neo Technology," | ||
| * Network Engine for Objects in Lund AB [http://neotechnology.com] | ||
| * | ||
| * This file is part of Neo4j. | ||
| * | ||
| * Neo4j is free software: you can redistribute it and/or modify | ||
| * it under the terms of the GNU General Public License as published by | ||
| * the Free Software Foundation, either version 3 of the License, or | ||
| * (at your option) any later version. | ||
| * | ||
| * This program is distributed in the hope that it will be useful, | ||
| * but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
| * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
| * GNU General Public License for more details. | ||
| * | ||
| * You should have received a copy of the GNU General Public License | ||
| * along with this program. If not, see <http://www.gnu.org/licenses/>. | ||
| */ | ||
| package org.neo4j.unsafe.impl.batchimport; | ||
|
|
||
| import org.neo4j.kernel.impl.store.record.RelationshipGroupRecord; | ||
| import org.neo4j.unsafe.impl.batchimport.staging.BatchSender; | ||
| import org.neo4j.unsafe.impl.batchimport.staging.Configuration; | ||
| import org.neo4j.unsafe.impl.batchimport.staging.ProcessorStep; | ||
| import org.neo4j.unsafe.impl.batchimport.staging.StageControl; | ||
|
|
||
| /** | ||
| * After this step is {@link #done()} all BOTH ID fields in the rel group cache will contain, | ||
| * for each node, the absolute number of groups from node 0 up to this point there is. | ||
| */ | ||
| public class CountGroupsStep extends ProcessorStep<RelationshipGroupRecord[]> | ||
| { | ||
| private final RelationshipGroupCache cache; | ||
|
|
||
| public CountGroupsStep( StageControl control, Configuration config, RelationshipGroupCache groupCache ) | ||
| { | ||
| super( control, "COUNT", config, 1 ); | ||
| this.cache = groupCache; | ||
| } | ||
|
|
||
| @Override | ||
| protected void process( RelationshipGroupRecord[] batch, BatchSender sender ) throws Throwable | ||
| { | ||
| for ( RelationshipGroupRecord group : batch ) | ||
| { | ||
| cache.incrementGroupCount( group.getOwningNode() ); | ||
| } | ||
| } | ||
| } |
Oops, something went wrong.