From c193d538fd8ac9d7b70ca2d64e151a076f6e5138 Mon Sep 17 00:00:00 2001 From: Marvin Froeder Date: Thu, 21 Jan 2016 13:33:56 +1300 Subject: [PATCH 01/11] Enabled build on windows --- appveyor.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 appveyor.yml diff --git a/appveyor.yml b/appveyor.yml new file mode 100644 index 00000000000..92eae9f465e --- /dev/null +++ b/appveyor.yml @@ -0,0 +1,23 @@ +version: '{build}' + +environment: + matrix: + - JAVA_HOME: C:\Program Files\Java\jdk1.8.0 + +os: Windows Server 2012 + +install: + - ps: | + Add-Type -AssemblyName System.IO.Compression.FileSystem + if (!(Test-Path -Path "C:\maven" )) { + (new-object System.Net.WebClient).DownloadFile('http://www.us.apache.org/dist/maven/maven-3/3.2.5/binaries/apache-maven-3.2.5-bin.zip', 'C:\maven-bin.zip') + [System.IO.Compression.ZipFile]::ExtractToDirectory("C:\maven-bin.zip", "C:\maven") + } + - cmd: SET PATH=C:\maven\apache-maven-3.2.5\bin;%JAVA_HOME%\bin;%PATH% + - cmd: SET MAVEN_OPTS=-XX:MaxPermSize=2g -Xmx4g + - cmd: SET JAVA_OPTS=-XX:MaxPermSize=2g -Xmx4g + - cmd: mvn --version + - cmd: java -version + +build_script: + - mvn clean install -B -Dci From 92edead8ba8c34516e4ecc3febcf60a8be8a6024 Mon Sep 17 00:00:00 2001 From: Marvin H Froeder Date: Thu, 21 Jan 2016 13:46:05 +1300 Subject: [PATCH 02/11] Ignoring rat on eclipse and appveyor files --- pom.xml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pom.xml b/pom.xml index 8f0bb2b6556..62eb81c93b4 100644 --- a/pom.xml +++ b/pom.xml @@ -264,6 +264,10 @@ limitations under the License. false .travis.yml + appveyor.yml + **/.classpath + **/.project + **/.settings/** .repository/** docs/static/** **/target/** From 0715c4ef247351cdd994378273f7d1b3aba282a8 Mon Sep 17 00:00:00 2001 From: Marvin Froeder Date: Mon, 1 Feb 2016 09:47:15 +1300 Subject: [PATCH 03/11] TINKERPOP-1041 - give import thread a chance to run --- .../tinkerpop/gremlin/groovy/engine/ScriptEnginesTest.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/gremlin-groovy/src/test/java/org/apache/tinkerpop/gremlin/groovy/engine/ScriptEnginesTest.java b/gremlin-groovy/src/test/java/org/apache/tinkerpop/gremlin/groovy/engine/ScriptEnginesTest.java index eb45c0badb3..9a7e1bf1a2f 100644 --- a/gremlin-groovy/src/test/java/org/apache/tinkerpop/gremlin/groovy/engine/ScriptEnginesTest.java +++ b/gremlin-groovy/src/test/java/org/apache/tinkerpop/gremlin/groovy/engine/ScriptEnginesTest.java @@ -185,6 +185,7 @@ public void shouldFailUntilImportExecutes() throws Exception { successes.incrementAndGet(); } catch (Exception ex) { if (failures.incrementAndGet() == 500) threadImport.start(); + Thread.yield(); } }) ); @@ -194,8 +195,8 @@ public void shouldFailUntilImportExecutes() throws Exception { threadEvalAndTriggerImport.join(); threadImport.join(); - assertTrue(successes.intValue() > 0); - assertTrue(failures.intValue() >= 500); + assertTrue("Success: " + successes.intValue() + " - Failures: " + failures.intValue(), successes.intValue() > 0); + assertTrue("Success: " + successes.intValue() + " - Failures: " + failures.intValue(), failures.intValue() >= 500); engines.close(); } From 9d0c87707e828b2ff5dbf8ef3d60051deebb055d Mon Sep 17 00:00:00 2001 From: Marvin Froeder Date: Mon, 1 Feb 2016 13:15:33 +1300 Subject: [PATCH 04/11] Url separator is always / even on windows --- .../org/apache/tinkerpop/gremlin/TestHelper.java | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/TestHelper.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/TestHelper.java index 303b1e3ec7c..a3c252fb50a 100644 --- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/TestHelper.java +++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/TestHelper.java @@ -32,6 +32,8 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; +import java.net.MalformedURLException; +import java.net.URISyntaxException; import java.net.URL; import java.util.Iterator; import java.util.List; @@ -50,6 +52,7 @@ public final class TestHelper { private static final String SEP = File.separator; + private static final char URL_SEP = '/'; public static final String TEST_DATA_RELATIVE_DIR = "test-case-data"; private TestHelper() { @@ -105,9 +108,14 @@ public static File getRootOfBuildDirectory(final Class clazz) { } private static String computePath(final Class clazz) { - final String clsUri = clazz.getName().replace('.', SEP.charAt(0)) + ".class"; + final String clsUri = clazz.getName().replace('.', URL_SEP) + ".class"; final URL url = clazz.getClassLoader().getResource(clsUri); - final String clsPath = url.getPath(); + String clsPath; + try { + clsPath = new File(url.toURI()).getAbsolutePath(); + } catch (URISyntaxException e) { + throw new RuntimeException("Unable to computePath for " + clazz, e); + } return clsPath.substring(0, clsPath.length() - clsUri.length()); } From daa6379bef196acdbdc769c2a516f1750010f4cc Mon Sep 17 00:00:00 2001 From: Marvin Froeder Date: Mon, 1 Feb 2016 17:19:33 +1300 Subject: [PATCH 05/11] Changed back slashes into regular ones --- .../tinkerpop/gremlin/hadoop/HadoopGraphProvider.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGraphProvider.java b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGraphProvider.java index 9600dad6391..4a6f06cdf5b 100644 --- a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGraphProvider.java +++ b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGraphProvider.java @@ -80,7 +80,7 @@ public class HadoopGraphProvider extends AbstractGraphProvider { "tinkerpop-classic.kryo", "tinkerpop-crew.kryo"); for (final String fileName : kryoResources) { - PATHS.put(fileName, TestHelper.generateTempFileFromResource(GryoResourceAccess.class, fileName, "").getAbsolutePath()); + PATHS.put(fileName, TestHelper.generateTempFileFromResource(GryoResourceAccess.class, fileName, "").getAbsolutePath().replace('\\', '/')); } final List graphsonResources = Arrays.asList( @@ -89,7 +89,7 @@ public class HadoopGraphProvider extends AbstractGraphProvider { "tinkerpop-classic.json", "tinkerpop-crew.json"); for (final String fileName : graphsonResources) { - PATHS.put(fileName, TestHelper.generateTempFileFromResource(GraphSONResourceAccess.class, fileName, "").getAbsolutePath()); + PATHS.put(fileName, TestHelper.generateTempFileFromResource(GraphSONResourceAccess.class, fileName, "").getAbsolutePath().replace('\\', '/')); } final List scriptResources = Arrays.asList( @@ -100,7 +100,7 @@ public class HadoopGraphProvider extends AbstractGraphProvider { "script-input-grateful-dead.groovy", "script-output-grateful-dead.groovy"); for (final String fileName : scriptResources) { - PATHS.put(fileName, TestHelper.generateTempFileFromResource(ScriptResourceAccess.class, fileName, "").getAbsolutePath()); + PATHS.put(fileName, TestHelper.generateTempFileFromResource(ScriptResourceAccess.class, fileName, "").getAbsolutePath().replace('\\', '/')); } } catch (Exception e) { e.printStackTrace(); From a6cfc3b822dc00107375f175003fb9379201b21d Mon Sep 17 00:00:00 2001 From: Marvin Froeder Date: Mon, 1 Feb 2016 17:19:55 +1300 Subject: [PATCH 06/11] Downloading hadoop winutils.exe when using windows --- pom.xml | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/pom.xml b/pom.xml index 62eb81c93b4..311ddad4ba2 100644 --- a/pom.xml +++ b/pom.xml @@ -1001,5 +1001,45 @@ limitations under the License. + + + windows + + false + + windows + + + + + + org.codehaus.mojo + wagon-maven-plugin + 1.0 + + + download-hadoop-winutils + generate-test-resources + + download-single + + + http://public-repo-1.hortonworks.com/ + hdp-win-alpha/winutils.exe + ${project.build.directory}/hadoop_home/bin + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + -Dlog4j.configuration=${log4j-test.properties} -Dbuild.dir=${project.build.directory} -Dhadoop.home.dir=${project.build.directory}/hadoop_home + + + + + From 317d5c8a2ca1677d634fc9a983eae234e6f2f323 Mon Sep 17 00:00:00 2001 From: Marvin Froeder Date: Mon, 1 Feb 2016 17:39:13 +1300 Subject: [PATCH 07/11] Changed back slashes into regular ones --- .../gremlin/spark/structure/io/SparkContextStorage.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/SparkContextStorage.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/SparkContextStorage.java index 76a3299719a..4b113a33294 100644 --- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/SparkContextStorage.java +++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/SparkContextStorage.java @@ -75,9 +75,9 @@ public List ls() { @Override public List ls(final String location) { final List rdds = new ArrayList<>(); - final String wildCardLocation = (location.endsWith("*") ? location : location + "*").replace(".", "\\.").replace("*", ".*"); + final String wildCardLocation = (location.endsWith("*") ? location : location + "*").replace(".", "\\.").replace("*", ".*").replace('\\', '/'); for (final RDD rdd : Spark.getRDDs()) { - if (rdd.name().matches(wildCardLocation)) + if (rdd.name().replace('\\', '/').matches(wildCardLocation)) rdds.add(rdd.name() + " [" + rdd.getStorageLevel().description() + "]"); } return rdds; @@ -103,9 +103,9 @@ public boolean exists(final String location) { @Override public boolean rm(final String location) { final List rdds = new ArrayList<>(); - final String wildCardLocation = (location.endsWith("*") ? location : location + "*").replace(".", "\\.").replace("*", ".*"); + final String wildCardLocation = (location.endsWith("*") ? location : location + "*").replace(".", "\\.").replace("*", ".*").replace('\\', '/'); for (final RDD rdd : Spark.getRDDs()) { - if (rdd.name().matches(wildCardLocation)) + if (rdd.name().replace('\\', '/').matches(wildCardLocation)) rdds.add(rdd.name()); } rdds.forEach(Spark::removeRDD); From 164fdace2e58e9632d8de4bba0fe8bea4c2bd5b2 Mon Sep 17 00:00:00 2001 From: Marvin Froeder Date: Wed, 3 Feb 2016 09:17:11 +1300 Subject: [PATCH 08/11] Fixed regex construction --- .../gremlin/spark/structure/io/SparkContextStorage.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/SparkContextStorage.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/SparkContextStorage.java index 4b113a33294..799c215792f 100644 --- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/SparkContextStorage.java +++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/SparkContextStorage.java @@ -75,7 +75,7 @@ public List ls() { @Override public List ls(final String location) { final List rdds = new ArrayList<>(); - final String wildCardLocation = (location.endsWith("*") ? location : location + "*").replace(".", "\\.").replace("*", ".*").replace('\\', '/'); + final String wildCardLocation = (location.endsWith("*") ? location : location + "*").replace('\\', '/').replace(".", "\\.").replace("*", ".*"); for (final RDD rdd : Spark.getRDDs()) { if (rdd.name().replace('\\', '/').matches(wildCardLocation)) rdds.add(rdd.name() + " [" + rdd.getStorageLevel().description() + "]"); @@ -103,7 +103,7 @@ public boolean exists(final String location) { @Override public boolean rm(final String location) { final List rdds = new ArrayList<>(); - final String wildCardLocation = (location.endsWith("*") ? location : location + "*").replace(".", "\\.").replace("*", ".*").replace('\\', '/'); + final String wildCardLocation = (location.endsWith("*") ? location : location + "*").replace('\\', '/').replace(".", "\\.").replace("*", ".*"); for (final RDD rdd : Spark.getRDDs()) { if (rdd.name().replace('\\', '/').matches(wildCardLocation)) rdds.add(rdd.name()); From 884e45d136d74144aa0d51c4ada3822bfc786ee9 Mon Sep 17 00:00:00 2001 From: Marvin Froeder Date: Thu, 4 Feb 2016 08:08:12 +1300 Subject: [PATCH 09/11] Attemp to download maven using https --- appveyor.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 92eae9f465e..e31d2ea285e 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -10,10 +10,10 @@ install: - ps: | Add-Type -AssemblyName System.IO.Compression.FileSystem if (!(Test-Path -Path "C:\maven" )) { - (new-object System.Net.WebClient).DownloadFile('http://www.us.apache.org/dist/maven/maven-3/3.2.5/binaries/apache-maven-3.2.5-bin.zip', 'C:\maven-bin.zip') + (new-object System.Net.WebClient).DownloadFile('https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.3.9/apache-maven-3.3.9-bin.zip', 'C:\maven-bin.zip') [System.IO.Compression.ZipFile]::ExtractToDirectory("C:\maven-bin.zip", "C:\maven") } - - cmd: SET PATH=C:\maven\apache-maven-3.2.5\bin;%JAVA_HOME%\bin;%PATH% + - cmd: SET PATH=C:\maven\apache-maven-3.3.9\bin;%JAVA_HOME%\bin;%PATH% - cmd: SET MAVEN_OPTS=-XX:MaxPermSize=2g -Xmx4g - cmd: SET JAVA_OPTS=-XX:MaxPermSize=2g -Xmx4g - cmd: mvn --version From bc58b8baa3debec17a076469894c0074afbaad1b Mon Sep 17 00:00:00 2001 From: Marvin Froeder Date: Mon, 15 Feb 2016 16:45:40 +1300 Subject: [PATCH 10/11] Installing maven using choco --- appveyor.yml | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index e31d2ea285e..a99752a7621 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -7,17 +7,8 @@ environment: os: Windows Server 2012 install: - - ps: | - Add-Type -AssemblyName System.IO.Compression.FileSystem - if (!(Test-Path -Path "C:\maven" )) { - (new-object System.Net.WebClient).DownloadFile('https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.3.9/apache-maven-3.3.9-bin.zip', 'C:\maven-bin.zip') - [System.IO.Compression.ZipFile]::ExtractToDirectory("C:\maven-bin.zip", "C:\maven") - } - - cmd: SET PATH=C:\maven\apache-maven-3.3.9\bin;%JAVA_HOME%\bin;%PATH% - - cmd: SET MAVEN_OPTS=-XX:MaxPermSize=2g -Xmx4g - - cmd: SET JAVA_OPTS=-XX:MaxPermSize=2g -Xmx4g - - cmd: mvn --version - - cmd: java -version + - cmd: choco install maven -y -f + - cmd: refreshenv build_script: - mvn clean install -B -Dci From 1ca95490d42892c4adffc867f8900e5e327e8812 Mon Sep 17 00:00:00 2001 From: Marvin Froeder Date: Sat, 13 Feb 2016 12:17:02 +1300 Subject: [PATCH 11/11] Created an appveyor profile that ignored knowly broken tests --- appveyor.yml | 2 +- spark-gremlin/pom.xml | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index a99752a7621..fe007405f39 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -11,4 +11,4 @@ install: - cmd: refreshenv build_script: - - mvn clean install -B -Dci + - mvn clean install -B -Dci -fae -Pappveyor diff --git a/spark-gremlin/pom.xml b/spark-gremlin/pom.xml index 7d6c666df89..fb7b2f74947 100644 --- a/spark-gremlin/pom.xml +++ b/spark-gremlin/pom.xml @@ -393,4 +393,25 @@ + + + + appveyor + + + + maven-surefire-plugin + + + **/SparkHadoopGremlinTest* + **/SparkGraphComputerGroovyProcessIntegrateTest* + **/SparkGraphComputerProcessIntegrateTest* + **/GryoSerializerIntegrateTest* + + + + + + + \ No newline at end of file