Skip to content
Permalink
Browse files
Fix date unequal since client and server has not same time zone (#192)
Change-Id: I708c3678df8ce64d5db49c40f8a3642738f00ca9
  • Loading branch information
Linary committed Dec 16, 2020
1 parent 784debb commit 7d2cfb775eec7262f320ee12f06be41570ad8c01
Show file tree
Hide file tree
Showing 4 changed files with 113 additions and 54 deletions.
@@ -138,7 +138,9 @@ protected FileLineFetcher createLineFetcher() {
private Configuration loadConfiguration() {
Configuration conf = new Configuration();
conf.addResource(new Path(this.source().coreSitePath()));
conf.addResource(new Path(this.source().hdfsSitePath()));
if (this.source().hdfsSitePath() != null) {
conf.addResource(new Path(this.source().hdfsSitePath()));
}
return conf;
}

@@ -55,13 +55,13 @@ public void check() throws IllegalArgumentException {
"The core site file '%s' is not an existing file",
coreSiteFile);

E.checkArgument(!StringUtils.isEmpty(this.hdfsSitePath),
"The hdfs_site_path can't be empty");
File hdfsSiteFile = FileUtils.getFile(Paths.get(this.hdfsSitePath)
.toString());
E.checkArgument(hdfsSiteFile.exists() && hdfsSiteFile.isFile(),
"The hdfs site file '%s' is not an existing file",
hdfsSiteFile);
if (this.hdfsSitePath != null) {
File hdfsSiteFile = FileUtils.getFile(Paths.get(this.hdfsSitePath)
.toString());
E.checkArgument(hdfsSiteFile.exists() && hdfsSiteFile.isFile(),
"The hdfs site file '%s' is not an existing file",
hdfsSiteFile);
}

if (this.kerberosConfig != null) {
this.kerberosConfig.check();
@@ -593,7 +593,8 @@ public void testValueListPropertyInJsonFile() {
}

@Test
public void testValueListPropertyInTextFile() {
public void testValueListPropertyInTextFile()
throws java.text.ParseException {
ioUtil.write("vertex_person.txt", "jin\t29\tBeijing");
ioUtil.write("vertex_software.txt", "tom\tChinese\t328");
ioUtil.write("edge_use.txt",
@@ -621,11 +622,12 @@ public void testValueListPropertyInTextFile() {
"2019-05-02 00:00:00.000",
"2008-05-02 00:00:00.000"
);
Assert.assertEquals(expectedTimes, edge.property("time"));
assertDateEquals(expectedTimes, edge.property("time"));
}

@Test
public void testValueSetPropertyInTextFile() {
public void testValueSetPropertyInTextFile()
throws java.text.ParseException {
ioUtil.write("vertex_person.txt", "jin\t29\tBeijing");
ioUtil.write("vertex_software.txt", "tom\tChinese\t328");
ioUtil.write("edge_use.txt",
@@ -650,9 +652,11 @@ public void testValueSetPropertyInTextFile() {
Assert.assertEquals(ImmutableList.of(4, 1, 5, 6),
edge.property("feel"));
Assert.assertEquals(ArrayList.class, edge.property("time").getClass());
List<?> list = (List<?>) edge.property("time");
Assert.assertTrue(list.contains("2019-05-02 00:00:00.000"));
Assert.assertTrue(list.contains("2008-05-02 00:00:00.000"));
List<String> expectedTimes = ImmutableList.of(
"2019-05-02 00:00:00.000",
"2008-05-02 00:00:00.000"
);
assertDateEquals(expectedTimes, edge.property("time"));
}

@Test
@@ -1376,14 +1380,14 @@ public void testTimestampAsDateFormat() throws java.text.ParseException {
Assert.assertEquals(2, vertices.size());

Vertex marko = vertices.get(0);
Assert.assertEquals("1992-10-01 00:00:00.000", marko.property("birth"));
assertDateEquals("1992-10-01 00:00:00.000", marko.property("birth"));

Vertex vadas = vertices.get(1);
Assert.assertEquals("2000-01-01 00:00:00.000", vadas.property("birth"));
assertDateEquals("2000-01-01 00:00:00.000", vadas.property("birth"));
}

@Test
public void testDefaultTimeZoneGMT8() {
public void testDefaultTimeZoneGMT8() throws java.text.ParseException {
ioUtil.write("vertex_person_birth_date.csv",
"marko,1992-10-01 12:00:00,Beijing",
"vadas,2000-01-01 13:00:00,Hongkong");
@@ -1401,16 +1405,14 @@ public void testDefaultTimeZoneGMT8() {
Assert.assertEquals(2, vertices.size());

Vertex marko = CLIENT.graph().getVertex("1:marko");
Assert.assertEquals("1992-10-01 12:00:00.000",
marko.property("birth"));
assertDateEquals("1992-10-01 12:00:00.000", marko.property("birth"));

Vertex vadas = CLIENT.graph().getVertex("1:vadas");
Assert.assertEquals("2000-01-01 13:00:00.000",
vadas.property("birth"));
assertDateEquals("2000-01-01 13:00:00.000", vadas.property("birth"));
}

@Test
public void testCustomizedTimeZoneGMT0() {
public void testCustomizedTimeZoneGMT0() throws java.text.ParseException {
ioUtil.write("vertex_person_birth_date.csv",
"marko,1992-10-01 12:00:00,Beijing",
"vadas,2000-01-01 13:00:00,Hongkong");
@@ -1428,16 +1430,16 @@ public void testCustomizedTimeZoneGMT0() {
Assert.assertEquals(2, vertices.size());

Vertex marko = CLIENT.graph().getVertex("1:marko");
Assert.assertEquals("1992-10-01 20:00:00.000",
marko.property("birth"));
assertDateEquals("1992-10-01 20:00:00.000",
marko.property("birth"));

Vertex vadas = CLIENT.graph().getVertex("1:vadas");
Assert.assertEquals("2000-01-01 21:00:00.000",
vadas.property("birth"));
assertDateEquals("2000-01-01 21:00:00.000",
vadas.property("birth"));
}

@Test
public void testValueMapping() {
public void testValueMapping() throws java.text.ParseException {
/*
* "age": {"1": 25, "2": 30}
* "birth": {"1": "1994-01-01", "2": "1989-01-01"}
@@ -1459,11 +1461,21 @@ public void testValueMapping() {
List<Vertex> vertices = CLIENT.graph().listVertices();
Assert.assertEquals(2, vertices.size());

// TODO: Fix date property be saved as long in client
assertContains(vertices, "person", "name", "marko", "age", 25,
"birth", "1994-01-01 00:00:00.000", "city", "Beijing");
assertContains(vertices, "person", "name", "vadas", "age", 30,
"birth", "1989-01-01 00:00:00.000", "city", "Shanghai");
Vertex marko, vadas;
if (vertices.get(0).property("name").equals("marko")) {
marko = vertices.get(0);
vadas = vertices.get(1);
} else {
vadas = vertices.get(0);
marko = vertices.get(1);
}
Assert.assertEquals(25, marko.property("age"));
Assert.assertEquals("Beijing", marko.property("city"));
assertDateEquals("1994-01-01 00:00:00.000", marko.property("birth"));

Assert.assertEquals(30, vadas.property("age"));
Assert.assertEquals("Shanghai", vadas.property("city"));
assertDateEquals("1989-01-01 00:00:00.000", vadas.property("birth"));
}

@Test
@@ -2315,7 +2327,7 @@ public void testSingleInsertEdgeWithCheckVertexFalse() {
}

@Test
public void testOrcCompressFile() {
public void testOrcCompressFile() throws java.text.ParseException {
// TODO: add test for blob and uuid
TypeInfo typeInfo = TypeInfoUtils.getTypeInfoFromTypeString(
"struct<" +
@@ -2355,8 +2367,7 @@ public void testOrcCompressFile() {
Assert.assertEquals(4.0D, vertex.property("p_float"));
Assert.assertEquals(5.0D, vertex.property("p_double"));
Assert.assertEquals("marko", vertex.property("p_string"));
Assert.assertEquals("2019-12-09 00:00:00.000",
vertex.property("p_date"));
assertDateEquals("2019-12-09 00:00:00.000", vertex.property("p_date"));
}

@Test
@@ -3010,12 +3021,17 @@ public void testHttpsClientValueMapping() {
};
HugeGraphLoader.main(args);

HugeClient httpsClient = HugeClient.builder(HTTPS_URL, GRAPH)
.configSSL(TRUST_STORE_FILE,
TRUST_STORE_PASSWORD)
.build();
List<Vertex> vertices = httpsClient.graph().listVertices();
Assert.assertEquals(2, vertices.size());
HugeClient httpsClient = null;
try {
httpsClient = HugeClient.builder(HTTPS_URL, GRAPH)
.configSSL(TRUST_STORE_FILE,
TRUST_STORE_PASSWORD)
.build();
List<Vertex> vertices = httpsClient.graph().listVertices();
Assert.assertEquals(2, vertices.size());
} finally {
clearAndClose(httpsClient, GRAPH);
}
}

@Test
@@ -3035,9 +3051,6 @@ public void testHttpsHolderClientValueMapping() {
"--batch-insert-threads", "2",
"--test-mode", "true"
};

String homePath = System.getProperty("loader.home.path");
System.setProperty("loader.home.path", "./");
HugeGraphLoader.main(args);

LoadOptions options = new LoadOptions();
@@ -3047,16 +3060,14 @@ public void testHttpsHolderClientValueMapping() {
options.protocol = HTTPS_PROTOCOL;
options.trustStoreFile = TRUST_STORE_FILE;
options.trustStorePassword = TRUST_STORE_PASSWORD;

HugeClient httpsClient = null;
try {
HugeClient client = HugeClientHolder.create(options);
List<Vertex> vertices = client.graph().listVertices();
httpsClient = HugeClientHolder.create(options);
List<Vertex> vertices = httpsClient.graph().listVertices();
Assert.assertEquals(2, vertices.size());
} finally {
if (homePath != null) {
System.setProperty("loader.home.path", homePath);
} else {
System.clearProperty("loader.home.path");
}
clearAndClose(httpsClient, GRAPH);
}
}
}
@@ -20,16 +20,18 @@
package com.baidu.hugegraph.loader.test.functional;

import java.nio.file.Paths;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.TimeZone;

import com.baidu.hugegraph.driver.HugeClient;
import com.baidu.hugegraph.structure.constant.T;
import com.baidu.hugegraph.structure.graph.Edge;
import com.baidu.hugegraph.structure.graph.Vertex;
import com.baidu.hugegraph.testutil.Assert;
import com.baidu.hugegraph.util.E;

public class LoadTest {

@@ -58,6 +60,13 @@ public static void clearServerData() {
CLIENT.graphs().clear(GRAPH, CONFIRM_CLEAR);
}

public static void clearAndClose(HugeClient httpsClient, String graph) {
if (httpsClient == null) {
return;
}
httpsClient.graphs().clear(graph, CONFIRM_CLEAR);
}

protected static void assertContains(List<Vertex> vertices, String label,
Object... keyValues) {
boolean matched = false;
@@ -91,8 +100,8 @@ protected static void assertContains(List<Edge> edges, String label,
}

private static Map<String, Object> toMap(Object... properties) {
E.checkArgument((properties.length & 0x01) == 0,
"The number of properties must be even");
Assert.assertTrue("The number of properties must be even",
(properties.length & 0x01) == 0);
Map<String, Object> map = new LinkedHashMap<>();
for (int i = 0; i < properties.length; i = i + 2) {
if (!properties[i].equals(T.id) && !properties[i].equals(T.label)) {
@@ -101,4 +110,41 @@ private static Map<String, Object> toMap(Object... properties) {
}
return map;
}

public static void assertDateEquals(String expectDate, Object actualDate)
throws java.text.ParseException {
Assert.assertEquals("Date value must be String class",
String.class, actualDate.getClass());
assertDateEquals(expectDate, TimeZone.getTimeZone("GMT+8"),
(String) actualDate, TimeZone.getDefault());
}

public static void assertDateEquals(List<String> expectDates,
Object actualDates)
throws java.text.ParseException {
Assert.assertTrue("Date value must be List<String> class",
List.class.isAssignableFrom(actualDates.getClass()));
List<String> actualDateList = (List<String>) actualDates;
Assert.assertEquals("The size of expect and actual dates must be equal",
expectDates.size(), actualDateList.size());
int n = expectDates.size();
for (int i = 0; i < n; i++) {
assertDateEquals(expectDates.get(i), TimeZone.getTimeZone("GMT+8"),
actualDateList.get(i), TimeZone.getDefault());
}
}

public static void assertDateEquals(String expectDate, TimeZone expectZone,
String actualDate, TimeZone actualZone)
throws java.text.ParseException {
DateFormat expectDF = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS");
expectDF.setTimeZone(expectZone);
long expectTimeStamp = expectDF.parse(expectDate).getTime();

DateFormat actualDF = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS");
actualDF.setTimeZone(actualZone);
long actualTimeStamp = actualDF.parse(actualDate).getTime();

Assert.assertEquals(expectTimeStamp, actualTimeStamp);
}
}

0 comments on commit 7d2cfb7

Please sign in to comment.