-
Notifications
You must be signed in to change notification settings - Fork 2.3k
/
ImportCommand.java
254 lines (234 loc) · 11.8 KB
/
ImportCommand.java
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
/*
* Copyright (c) 2002-2016 "Neo Technology,"
* Network Engine for Objects in Lund AB [http://neotechnology.com]
*
* This file is part of Neo4j.
*
* Neo4j is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package org.neo4j.commandline.dbms;
import java.io.IOException;
import java.nio.file.Path;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import org.neo4j.commandline.admin.AdminCommand;
import org.neo4j.commandline.admin.CommandFailed;
import org.neo4j.commandline.admin.IncorrectUsage;
import org.neo4j.commandline.admin.OutsideWorld;
import org.neo4j.commandline.arguments.Arguments;
import org.neo4j.commandline.arguments.MandatoryNamedArg;
import org.neo4j.commandline.arguments.OptionalNamedArg;
import org.neo4j.dbms.DatabaseManagementSystemSettings;
import org.neo4j.graphdb.factory.GraphDatabaseSettings;
import org.neo4j.helpers.Args;
import org.neo4j.helpers.collection.MapUtil;
import org.neo4j.kernel.configuration.Config;
import org.neo4j.kernel.impl.util.Validators;
import org.neo4j.server.configuration.ConfigLoader;
public class ImportCommand implements AdminCommand
{
public static final String DEFAULT_REPORT_FILE_NAME = "import.report";
private static final String[] allowedModes = {"database", "csv"};
private static final Arguments databaseArguments = new Arguments()
.withArgument( new MandatoryNamedArg( "mode", "database", "Import a pre-3.0 installation." ) {
@Override
public String usage()
{
return String.format( "--%s=%s", name(), exampleValue() );
}
} )
.withDatabase()
.withAdditionalConfig()
.withArgument( new OptionalNamedArg( "from", "source-directory", "",
"The location of the pre-3.0 database (e.g. <neo4j-root>/data/graph.db)." ) );
private static final Arguments csvArguments = new Arguments()
.withArgument( new OptionalNamedArg( "mode", "csv", "csv", "Import a collection of CSV files." ) {
@Override
public String usage()
{
return String.format( "[--%s=%s]", name(), exampleValue() );
}
} )
.withDatabase()
.withAdditionalConfig()
.withArgument( new OptionalNamedArg( "report-file", "filename", DEFAULT_REPORT_FILE_NAME,
"File in which to store the report of the csv-import." ) )
.withArgument( new OptionalNamedArg( "nodes[:Label1:Label2]", "\"file1,file2,...\"", "",
"Node CSV header and data. Multiple files will be logically seen as " +
"one big file from the perspective of the importer. The first line " +
"must contain the header. Multiple data sources like these can be " +
"specified in one import, where each data source has its own header. " +
"Note that file groups must be enclosed in quotation marks." ) )
.withArgument( new OptionalNamedArg( "relationships[:RELATIONSHIP_TYPE]", "\"file1,file2,...\"", "",
"Relationship CSV header and data. Multiple files will be logically " +
"seen as one big file from the perspective of the importer. The first " +
"line must contain the header. Multiple data sources like these can be " +
"specified in one import, where each data source has its own header. " +
"Note that file groups must be enclosed in quotation marks." ) )
.withArgument( new OptionalNamedArg( "id-type", new String[]{"STRING", "INTEGER", "ACTUAL"},
"STRING", "Each node must provide a unique id. This is used to find the correct " +
"nodes when creating relationships. Possible values are " +
"STRING: arbitrary strings for identifying nodes, " +
"INTEGER: arbitrary integer values for identifying nodes, " +
"ACTUAL: (advanced) actual node ids. " +
"For more information on id handling, please see the Neo4j Manual: " +
"http://neo4j.com/docs/operations-manual/current/deployment/#import-tool" ) )
.withArgument( new OptionalNamedArg( "input-encoding", "character-set", "UTF-8",
"Character set that input data is encoded in." ) );
private static final Arguments allArguments = new Arguments()
.withDatabase()
.withAdditionalConfig()
.withArgument( new OptionalNamedArg( "mode", allowedModes, "csv",
"Import a collection of CSV files or a pre-3.0 installation." ) )
.withArgument( new OptionalNamedArg( "from", "source-directory", "",
"The location of the pre-3.0 database (e.g. <neo4j-root>/data/graph.db)." ) )
.withArgument( new OptionalNamedArg( "report-file", "filename", DEFAULT_REPORT_FILE_NAME,
"File in which to store the report of the csv-import." ) )
.withArgument( new OptionalNamedArg( "nodes[:Label1:Label2]", "\"file1,file2,...\"", "",
"Node CSV header and data. Multiple files will be logically seen as " +
"one big file from the perspective of the importer. The first line " +
"must contain the header. Multiple data sources like these can be " +
"specified in one import, where each data source has its own header. " +
"Note that file groups must be enclosed in quotation marks." ) )
.withArgument( new OptionalNamedArg( "relationships[:RELATIONSHIP_TYPE]", "\"file1,file2,...\"", "",
"Relationship CSV header and data. Multiple files will be logically " +
"seen as one big file from the perspective of the importer. The first " +
"line must contain the header. Multiple data sources like these can be " +
"specified in one import, where each data source has its own header. " +
"Note that file groups must be enclosed in quotation marks." ) )
.withArgument( new OptionalNamedArg( "id-type", new String[]{"STRING", "INTEGER", "ACTUAL"},
"STRING", "Each node must provide a unique id. This is used to find the correct " +
"nodes when creating relationships. Possible values are " +
"STRING: arbitrary strings for identifying nodes, " +
"INTEGER: arbitrary integer values for identifying nodes, " +
"ACTUAL: (advanced) actual node ids. " +
"For more information on id handling, please see the Neo4j Manual: " +
"http://neo4j.com/docs/operations-manual/current/deployment/#import-tool" ) )
.withArgument( new OptionalNamedArg( "input-encoding", "character-set", "UTF-8",
"Character set that input data is encoded in." ) );
public static class Provider extends AdminCommand.Provider
{
public Provider()
{
super( "import" );
}
@Override
public Arguments allArguments()
{
return allArguments;
}
@Override
public List<Arguments> possibleArguments()
{
return Arrays.asList( csvArguments, databaseArguments );
}
@Override
public String description()
{
return "Import a collection of CSV files with --mode=csv (default), or a database from " +
"a pre-3.0 installation with --mode=database.";
}
@Override
public String summary()
{
return "Import from a collection of CSV files or a pre-3.0 database.";
}
@Override
public AdminCommand create( Path homeDir, Path configDir, OutsideWorld outsideWorld )
{
return new ImportCommand( homeDir, configDir, outsideWorld );
}
}
private final Path homeDir;
private final Path configDir;
private final OutsideWorld outsideWorld;
private final ImporterFactory importerFactory;
public ImportCommand( Path homeDir, Path configDir, OutsideWorld outsideWorld )
{
this( homeDir, configDir, outsideWorld, new ImporterFactory() );
}
ImportCommand( Path homeDir, Path configDir, OutsideWorld outsideWorld, ImporterFactory importerFactory )
{
this.homeDir = homeDir;
this.configDir = configDir;
this.outsideWorld = outsideWorld;
this.importerFactory = importerFactory;
}
@Override
public void execute( String[] args ) throws IncorrectUsage, CommandFailed
{
String mode;
Optional<Path> additionalConfigFile;
String database;
try
{
mode = allArguments.parse("mode", args);
database = allArguments.parse( "database", args );
additionalConfigFile = allArguments.parseOptionalPath( "additional-config", args );
}
catch ( IllegalArgumentException e )
{
throw new IncorrectUsage( e.getMessage() );
}
try
{
Config config =
loadNeo4jConfig( homeDir, configDir, database, loadAdditionalConfig( additionalConfigFile ) );
Validators.CONTAINS_NO_EXISTING_DATABASE
.validate( config.get( DatabaseManagementSystemSettings.database_path ) );
Importer importer = importerFactory.getImporterForMode( mode, Args.parse( args ), config, outsideWorld );
importer.doImport();
}
catch ( IllegalArgumentException e )
{
throw new IncorrectUsage( e.getMessage() );
}
catch ( IOException e )
{
throw new RuntimeException( e );
}
}
private Map<String,String> loadAdditionalConfig( Optional<Path> additionalConfigFile )
{
if ( additionalConfigFile.isPresent() )
{
try
{
return MapUtil.load( additionalConfigFile.get().toFile() );
}
catch ( IOException e )
{
throw new IllegalArgumentException(
String.format( "Could not read configuration file [%s]", additionalConfigFile ), e );
}
}
return new HashMap<>();
}
private static Config loadNeo4jConfig( Path homeDir, Path configDir, String databaseName,
Map<String,String> additionalConfig )
{
ConfigLoader configLoader = new ConfigLoader( settings() );
Config config = configLoader.loadOfflineConfig( Optional.of( homeDir.toFile() ),
Optional.of( configDir.resolve( "neo4j.conf" ).toFile() ) );
additionalConfig.put( DatabaseManagementSystemSettings.active_database.name(), databaseName );
return config.with( additionalConfig );
}
private static List<Class<?>> settings()
{
return Arrays.asList( GraphDatabaseSettings.class, DatabaseManagementSystemSettings.class );
}
}