Skip to content
Permalink
Browse files
Support customized list element delimiter in text file (#66)
Implement #65
Change-Id: I9c20483c38133a04ce20e9a0843a2dce98d1ec73
  • Loading branch information
Linary authored and javeme committed Apr 25, 2019
1 parent 3a5e2f6 commit 0ef50dc5132bf7ec61bfc59dd7aed72c84b16cad
Show file tree
Hide file tree
Showing 15 changed files with 411 additions and 86 deletions.
@@ -208,12 +208,7 @@ protected void checkVertexIdLength(String id) {
protected Object validatePropertyValue(String key, Object rawValue) {
PropertyKey pKey = this.getPropertyKey(key);
InputSource inputSource = this.source().input();
Object value = DataTypeUtil.convert(rawValue, pKey, inputSource);
E.checkArgument(value != null,
"The value '%s' can't convert to class %s " +
"with cardinality %s",
rawValue, pKey.dataType().clazz(), pKey.cardinality());
return value;
return DataTypeUtil.convert(rawValue, pKey, inputSource);
}

protected static long parseNumberId(Object idValue) {
@@ -28,7 +28,7 @@
import com.fasterxml.jackson.databind.node.JsonNodeType;

public class FileSourceDeserializer extends
InputSourceDeserializer<FileSource> {
InputSourceDeserializer<FileSource> {

private static final String FIELD_FILE_PATH = "path";
private static final String FIELD_FILE_FORMAT = "format";
@@ -0,0 +1,37 @@
/*
* Copyright 2017 HugeGraph Authors
*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/

package com.baidu.hugegraph.loader.source;

import com.baidu.hugegraph.loader.source.file.ListFormat;
import com.fasterxml.jackson.annotation.JsonProperty;

public abstract class AbstractSource implements InputSource {

@JsonProperty("list_format")
private ListFormat listFormat;

public AbstractSource() {
this.listFormat = new ListFormat();
}

public ListFormat listFormat() {
return this.listFormat;
}
}
@@ -22,11 +22,11 @@
import java.util.Collections;
import java.util.List;

import com.baidu.hugegraph.loader.source.InputSource;
import com.baidu.hugegraph.loader.source.AbstractSource;
import com.baidu.hugegraph.loader.source.SourceType;
import com.fasterxml.jackson.annotation.JsonProperty;

public class FileSource implements InputSource {
public class FileSource extends AbstractSource {

private static final String DEFAULT_CHARSET = "UTF-8";
private static final String DEFAULT_DATE_FORMAT = "yyyy-MM-dd HH:mm:ss";
@@ -78,6 +78,14 @@ public List<String> header() {
}

public String delimiter() {
// TODO: Choose a better timing check, will be implemnted in 'value mapping' commit
if (this.delimiter != null &&
this.delimiter.equals(this.listFormat().elemDelimiter())) {
throw new IllegalStateException(String.format(
"The delimiter '%s' can't equal with collection " +
"elem delimiter '%s'", this.delimiter,
this.listFormat().elemDelimiter()));
}
return this.delimiter;
}

@@ -0,0 +1,65 @@
/*
* Copyright 2017 HugeGraph Authors
*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/

package com.baidu.hugegraph.loader.source.file;

import java.util.Collections;
import java.util.HashSet;
import java.util.Set;

import com.fasterxml.jackson.annotation.JsonProperty;

public final class ListFormat {

private static final String DEFAULT_START_SYMBOL = "[";
private static final String DEFAULT_END_SYMBOL = "]";
private static final String DEFAULT_ELEM_DELIMITER = ",";

@JsonProperty("start_symbol")
private String startSymbol;
@JsonProperty("end_symbol")
private String endSymbol;
@JsonProperty("elem_delimiter")
private String elemDelimiter;
@JsonProperty("ignored_elems")
private Set<String> ignoredElems;

public ListFormat() {
this.startSymbol = DEFAULT_START_SYMBOL;
this.endSymbol = DEFAULT_END_SYMBOL;
this.elemDelimiter = DEFAULT_ELEM_DELIMITER;
this.ignoredElems = new HashSet<>();
}

public String startSymbol() {
return this.startSymbol;
}

public String endSymbol() {
return this.endSymbol;
}

public String elemDelimiter() {
return this.elemDelimiter;
}

public Set<String> ignoredElems() {
return Collections.unmodifiableSet(this.ignoredElems);
}
}
@@ -19,11 +19,11 @@

package com.baidu.hugegraph.loader.source.jdbc;

import com.baidu.hugegraph.loader.source.InputSource;
import com.baidu.hugegraph.loader.source.AbstractSource;
import com.baidu.hugegraph.loader.source.SourceType;
import com.fasterxml.jackson.annotation.JsonProperty;

public class JDBCSource implements InputSource {
public class JDBCSource extends AbstractSource {

@JsonProperty("driver")
private String driver;

0 comments on commit 0ef50dc

Please sign in to comment.