Skip to content
Permalink
Browse files
Support skip struct to avoid delete it (#101)
implement #100

Change-Id: Ie7fb4a11e17e957b111dddb972845f444a5e8053
  • Loading branch information
Linary authored and javeme committed Nov 7, 2019
1 parent 5bdbc73 commit db3327b82eaeb5edc71d869153df93aa2d9fc314
Showing 5 changed files with 130 additions and 1 deletion.
@@ -147,6 +147,9 @@ private void load(ElemType type, List<ElementStruct> structs) {
LoadSummary summary = this.context.summary();
InputProgressMap newProgress = this.context.newProgress().type(type);
for (ElementStruct struct : structs) {
if (struct.skip()) {
continue;
}
StopWatch loadTimer = StopWatch.createStarted();
LoadMetrics metrics = summary.metrics(struct);
if (!this.context.stopped()) {
@@ -30,7 +30,6 @@
import com.baidu.hugegraph.loader.constant.ElemType;
import com.baidu.hugegraph.loader.constant.Unique;
import com.baidu.hugegraph.loader.source.InputSource;
import com.baidu.hugegraph.loader.source.file.FileSource;
import com.baidu.hugegraph.loader.util.JsonUtil;
import com.baidu.hugegraph.util.E;
import com.baidu.hugegraph.util.HashUtil;
@@ -41,6 +40,8 @@ public abstract class ElementStruct implements Unique<String>, Checkable {

@JsonProperty("label")
private String label;
@JsonProperty("skip")
private boolean skip;
@JsonProperty("input")
private InputSource input;
@JsonProperty("field_mapping")
@@ -59,6 +60,7 @@ public abstract class ElementStruct implements Unique<String>, Checkable {
private transient String uniqueKey;

public ElementStruct() {
this.skip = false;
this.mappingFields = new HashMap<>();
this.mappingValues = new HashMap<>();
this.selectedFields = new HashSet<>();
@@ -121,6 +123,10 @@ public String label() {
return this.label;
}

public boolean skip() {
return this.skip;
}

public InputSource input() {
return this.input;
}
@@ -240,6 +240,51 @@ public void testNoSchemaFile() {
});
}

@Test
public void testSkipStruct() {
ioUtil.write("vertex_person.csv",
"name,age,city",
"marko,29,Beijing",
"vadas,27,Hongkong",
"josh,32,Beijing",
"peter,35,Shanghai",
"\"li,nary\",26,\"Wu,han\"");
ioUtil.write("vertex_software.csv", GBK,
"name,lang,price",
"lop,java,328",
"ripple,java,199");
ioUtil.write("edge_knows.csv",
"source_name,target_name,date,weight",
"marko,vadas,20160110,0.5",
"marko,josh,20130220,1.0");
ioUtil.write("edge_created.csv",
"source_name,target_name,date,weight",
"marko,lop,20171210,0.4",
"josh,lop,20091111,0.4",
"josh,ripple,20171210,1.0",
"peter,lop,20170324,0.2");

String[] args = new String[]{
"-f", configPath("skip_struct/struct.json"),
"-s", configPath("skip_struct/schema.groovy"),
"-g", GRAPH,
"-h", SERVER,
"--num-threads", "2",
"--test-mode", "true"
};
HugeGraphLoader.main(args);

List<Vertex> vertices = CLIENT.graph().listVertices();
List<Edge> edges = CLIENT.graph().listEdges();

Assert.assertEquals(7, vertices.size());
Assert.assertEquals(4, edges.size());

for (Edge edge : edges) {
Assert.assertEquals("created", edge.label());
}
}

@Test
public void testVertexIdExceedLimit() {
Integer[] array = new Integer[129];
@@ -0,0 +1,17 @@
// Define schema
schema.propertyKey("name").asText().ifNotExist().create();
schema.propertyKey("age").asInt().ifNotExist().create();
schema.propertyKey("city").asText().ifNotExist().create();
schema.propertyKey("weight").asDouble().ifNotExist().create();
schema.propertyKey("lang").asText().ifNotExist().create();
schema.propertyKey("date").asText().ifNotExist().create();
schema.propertyKey("price").asDouble().ifNotExist().create();
schema.propertyKey("feel").asText().valueList().ifNotExist().create();
schema.propertyKey("time").asText().valueSet().ifNotExist().create();

schema.vertexLabel("person").properties("name", "age", "city").primaryKeys("name").ifNotExist().create();
schema.vertexLabel("software").properties("name", "lang", "price").primaryKeys("name").ifNotExist().create();

schema.edgeLabel("knows").sourceLabel("person").targetLabel("person").properties("date", "weight").ifNotExist().create();
schema.edgeLabel("created").sourceLabel("person").targetLabel("software").properties("date", "weight").ifNotExist().create();
schema.edgeLabel("use").sourceLabel("person").targetLabel("software").properties("feel", "time").nullableKeys("feel", "time").ifNotExist().create();
@@ -0,0 +1,58 @@
{
"vertices": [
{
"label": "person",
"input": {
"type": "${source_type}",
"path": "${store_path}/vertex_person.csv",
"format": "CSV",
"charset": "UTF-8"
},
"field_mapping": {
"name": "name",
"age": "age",
"city": "city"
}
},
{
"label": "software",
"input": {
"type": "${source_type}",
"path": "${store_path}/vertex_software.csv",
"format": "CSV"
}
}
],
"edges": [
{
"label": "knows",
"skip": true,
"source": ["source_name"],
"target": ["target_name"],
"input": {
"type": "${source_type}",
"path": "${store_path}/edge_knows.csv",
"format": "CSV"
},
"field_mapping": {
"source_name": "name",
"target_name": "name"
}
},
{
"label": "created",
"skip": false,
"source": ["source_name"],
"target": ["target_name"],
"input": {
"type": "${source_type}",
"path": "${store_path}/edge_created.csv",
"format": "CSV"
},
"field_mapping": {
"source_name": "name",
"target_name": "name"
}
}
]
}

0 comments on commit db3327b

Please sign in to comment.