-
Notifications
You must be signed in to change notification settings - Fork 2.3k
/
Configuration.java
163 lines (143 loc) · 5.26 KB
/
Configuration.java
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
/*
* Copyright (c) 2002-2016 "Neo Technology,"
* Network Engine for Objects in Lund AB [http://neotechnology.com]
*
* This file is part of Neo4j.
*
* Neo4j is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package org.neo4j.unsafe.impl.batchimport;
import org.neo4j.kernel.configuration.Config;
import org.neo4j.kernel.impl.pagecache.ConfiguringPageCacheFactory;
import static java.lang.Math.min;
import static org.neo4j.graphdb.factory.GraphDatabaseSettings.dense_node_threshold;
import static org.neo4j.io.ByteUnit.kibiBytes;
import static org.neo4j.io.ByteUnit.mebiBytes;
import static org.neo4j.kernel.configuration.Settings.BYTES;
/**
* User controlled configuration for a {@link BatchImporter}.
*/
public interface Configuration extends org.neo4j.unsafe.impl.batchimport.staging.Configuration
{
/**
* File name in which bad entries from the import will end up. This file will be created in the
* database directory of the imported database, i.e. <into>/bad.log.
*/
String BAD_FILE_NAME = "bad.log";
long MAX_PAGE_CACHE_MEMORY = mebiBytes( 240 );
/**
* @return number of relationships threshold for considering a node dense.
*/
int denseNodeThreshold();
/**
* @return amount of memory to reserve for the page cache. This should just be "enough" for it to be able
* to sequentially read and write a couple of stores at a time. If configured too high then there will
* be less memory available for other caches which are critical during the import. Optimal size is
* estimated to be 100-200 MiB. The importer will figure out an optimal page size from this value,
* with slightly bigger page size than "normal" random access use cases.
*/
long pageCacheMemory();
int pageSize();
class Default
extends org.neo4j.unsafe.impl.batchimport.staging.Configuration.Default
implements Configuration
{
@Override
public long pageCacheMemory()
{
// Get the upper bound of what we can get from the default config calculation
// We even want to limit amount of memory a bit more since we don't need very much during import
String defaultPageCacheMemory = ConfiguringPageCacheFactory.defaultHeuristicPageCacheMemory();
return min( MAX_PAGE_CACHE_MEMORY, BYTES.apply( defaultPageCacheMemory ) );
}
@Override
public int denseNodeThreshold()
{
return Integer.parseInt( dense_node_threshold.getDefaultValue() );
}
private static int calculateOptimalPageSize( long memorySize, int numberOfPages )
{
int pageSize = (int) mebiBytes( 8 );
int lowest = (int) kibiBytes( 8 );
while ( pageSize > lowest )
{
if ( memorySize / pageSize >= numberOfPages )
{
return pageSize;
}
pageSize >>>= 1;
}
return lowest;
}
@Override
public int pageSize()
{
return calculateOptimalPageSize( pageCacheMemory(), 60 );
}
}
Configuration DEFAULT = new Default();
class Overridden
extends org.neo4j.unsafe.impl.batchimport.staging.Configuration.Overridden
implements Configuration
{
private final Configuration defaults;
private final Config config;
public Overridden( Configuration defaults )
{
this( defaults, Config.empty() );
}
public Overridden( Configuration defaults, Config config )
{
super( defaults );
this.defaults = defaults;
this.config = config;
}
public Overridden( Config config )
{
this( Configuration.DEFAULT, config );
}
@Override
public long pageCacheMemory()
{
String defaultPageCacheMemory = ConfiguringPageCacheFactory.defaultHeuristicPageCacheMemory();
return min( MAX_PAGE_CACHE_MEMORY, BYTES.apply( defaultPageCacheMemory ) );
}
@Override
public int denseNodeThreshold()
{
return config.get( dense_node_threshold );
}
@Override
public int movingAverageSize()
{
return defaults.movingAverageSize();
}
@Override
public int pageSize()
{
return defaults.pageSize();
}
}
public static Configuration withBatchSize( Configuration config, int batchSize )
{
return new Overridden( config )
{
@Override
public int batchSize()
{
return batchSize;
}
};
}
}