Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
for just getting a dataset of size X imported as quickly as possible. Uses CsvDataGenerator for generating very crude, random data of varying sizes. Focuses on size more than layout of the data. QuickImport uses CsvDataGenerator as Input to BatchImporter for short-circuting the data generator and importer. Made refactorings around InputEntityDeserializer (CharSeeker->InputEntity) to allow for composition and reuse of code for this purpose.
- Loading branch information
Showing
13 changed files
with
845 additions
and
260 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
98 changes: 98 additions & 0 deletions
98
community/import-tool/src/test/java/org/neo4j/tooling/CsvDataGeneratorInput.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,98 @@ | ||
| /** | ||
| * Copyright (c) 2002-2015 "Neo Technology," | ||
| * Network Engine for Objects in Lund AB [http://neotechnology.com] | ||
| * | ||
| * This file is part of Neo4j. | ||
| * | ||
| * Neo4j is free software: you can redistribute it and/or modify | ||
| * it under the terms of the GNU General Public License as published by | ||
| * the Free Software Foundation, either version 3 of the License, or | ||
| * (at your option) any later version. | ||
| * | ||
| * This program is distributed in the hope that it will be useful, | ||
| * but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
| * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
| * GNU General Public License for more details. | ||
| * | ||
| * You should have received a copy of the GNU General Public License | ||
| * along with this program. If not, see <http://www.gnu.org/licenses/>. | ||
| */ | ||
| package org.neo4j.tooling; | ||
|
|
||
| import org.neo4j.unsafe.impl.batchimport.BatchImporter; | ||
| import org.neo4j.unsafe.impl.batchimport.InputIterable; | ||
| import org.neo4j.unsafe.impl.batchimport.InputIterator; | ||
| import org.neo4j.unsafe.impl.batchimport.cache.idmapping.IdGenerator; | ||
| import org.neo4j.unsafe.impl.batchimport.cache.idmapping.IdMapper; | ||
| import org.neo4j.unsafe.impl.batchimport.input.Groups; | ||
| import org.neo4j.unsafe.impl.batchimport.input.Input; | ||
| import org.neo4j.unsafe.impl.batchimport.input.InputNode; | ||
| import org.neo4j.unsafe.impl.batchimport.input.InputRelationship; | ||
| import org.neo4j.unsafe.impl.batchimport.input.csv.Configuration; | ||
| import org.neo4j.unsafe.impl.batchimport.input.csv.Header; | ||
| import org.neo4j.unsafe.impl.batchimport.input.csv.IdType; | ||
| import org.neo4j.unsafe.impl.batchimport.input.csv.InputNodeDeserialization; | ||
| import org.neo4j.unsafe.impl.batchimport.input.csv.InputRelationshipDeserialization; | ||
|
|
||
| /** | ||
| * Uses {@link CsvDataGenerator} as an {@link Input} directly into a {@link BatchImporter}. | ||
| */ | ||
| public class CsvDataGeneratorInput extends CsvDataGenerator<InputNode,InputRelationship> implements Input | ||
| { | ||
| private final IdType idType; | ||
|
|
||
| public CsvDataGeneratorInput( Header nodeHeader, Header relationshipHeader, | ||
| Configuration config, long nodes, long relationships, Groups groups, IdType idType, | ||
| int numberOfLabels, int numberOfRelationshipTypes ) | ||
| { | ||
| super( nodeHeader, relationshipHeader, config, nodes, relationships, | ||
| new InputNodeDeserialization( nodeHeader, groups, idType.idsAreExternal() ), | ||
| new InputRelationshipDeserialization( relationshipHeader, groups ), | ||
| numberOfLabels, numberOfRelationshipTypes ); | ||
| this.idType = idType; | ||
| } | ||
|
|
||
| @Override | ||
| public InputIterable<InputNode> nodes() | ||
| { | ||
| return new InputIterable<InputNode>() | ||
| { | ||
| @Override | ||
| public InputIterator<InputNode> iterator() | ||
| { | ||
| return nodeData(); | ||
| } | ||
| }; | ||
| } | ||
|
|
||
| @Override | ||
| public InputIterable<InputRelationship> relationships() | ||
| { | ||
| return new InputIterable<InputRelationship>() | ||
| { | ||
| @Override | ||
| public InputIterator<InputRelationship> iterator() | ||
| { | ||
| return relationshipData(); | ||
| } | ||
| }; | ||
| } | ||
|
|
||
| @Override | ||
| public IdMapper idMapper() | ||
| { | ||
| return idType.idMapper(); | ||
| } | ||
|
|
||
| @Override | ||
| public IdGenerator idGenerator() | ||
| { | ||
| return idType.idGenerator(); | ||
| } | ||
|
|
||
| @Override | ||
| public boolean specificRelationshipIds() | ||
| { | ||
| return false; | ||
| } | ||
| } |
Oops, something went wrong.