-
Notifications
You must be signed in to change notification settings - Fork 2.3k
/
ParallelBatchImporter.java
98 lines (90 loc) · 4.23 KB
/
ParallelBatchImporter.java
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
/*
* Copyright (c) 2002-2018 "Neo Technology,"
* Network Engine for Objects in Lund AB [http://neotechnology.com]
*
* This file is part of Neo4j.
*
* Neo4j is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package org.neo4j.unsafe.impl.batchimport;
import java.io.File;
import java.io.IOException;
import org.neo4j.io.fs.FileSystemAbstraction;
import org.neo4j.io.pagecache.PageCache;
import org.neo4j.kernel.configuration.Config;
import org.neo4j.kernel.impl.logging.LogService;
import org.neo4j.kernel.impl.store.format.RecordFormats;
import org.neo4j.kernel.lifecycle.LifecycleAdapter;
import org.neo4j.unsafe.impl.batchimport.input.Input;
import org.neo4j.unsafe.impl.batchimport.staging.ExecutionMonitor;
import org.neo4j.unsafe.impl.batchimport.store.BatchingNeoStores;
import static org.neo4j.unsafe.impl.batchimport.ImportLogic.instantiateNeoStores;
/**
* {@link BatchImporter} which tries to exercise as much of the available resources to gain performance.
* Or rather ensure that the slowest resource (usually I/O) is fully saturated and that enough work is
* being performed to keep that slowest resource saturated all the time.
* <p>
* Overall goals: split up processing cost by parallelizing. Keep CPUs busy, keep I/O busy and writing sequentially.
* I/O is only allowed to be read to and written from sequentially, any random access drastically reduces performance.
* Goes through multiple stages where each stage has one or more steps executing in parallel, passing
* batches between these steps through each stage, i.e. passing batches downstream.
*/
public class ParallelBatchImporter extends LifecycleAdapter implements BatchImporter
{
private final PageCache externalPageCache;
private final File storeDir;
private final FileSystemAbstraction fileSystem;
private final Configuration config;
private final LogService logService;
private final Config dbConfig;
private final RecordFormats recordFormats;
private final ExecutionMonitor executionMonitor;
private final AdditionalInitialIds additionalInitialIds;
private final ImportLogic.Monitor monitor;
public ParallelBatchImporter( File storeDir, FileSystemAbstraction fileSystem, PageCache externalPageCache,
Configuration config, LogService logService, ExecutionMonitor executionMonitor,
AdditionalInitialIds additionalInitialIds, Config dbConfig, RecordFormats recordFormats, ImportLogic.Monitor monitor )
{
this.externalPageCache = externalPageCache;
this.storeDir = storeDir;
this.fileSystem = fileSystem;
this.config = config;
this.logService = logService;
this.dbConfig = dbConfig;
this.recordFormats = recordFormats;
this.executionMonitor = executionMonitor;
this.additionalInitialIds = additionalInitialIds;
this.monitor = monitor;
}
@Override
public void doImport( Input input ) throws IOException
{
try ( BatchingNeoStores store = instantiateNeoStores( fileSystem, storeDir, externalPageCache, recordFormats,
config, logService, additionalInitialIds, dbConfig );
ImportLogic logic = new ImportLogic( storeDir, fileSystem, store, config, logService,
executionMonitor, recordFormats, monitor ) )
{
store.createNew();
logic.initialize( input );
logic.importNodes();
logic.prepareIdMapper();
logic.importRelationships();
logic.calculateNodeDegrees();
logic.linkRelationshipsOfAllTypes();
logic.defragmentRelationshipGroups();
logic.buildCountsStore();
store.success();
}
}
}