Skip to content
Permalink
Browse files
BIGTOP-3137: Giraph smoke tests implementation
Add smoke tests for Giraph

Change-Id: I80ec1fbc916d5f616e92a16a15db777c4f4d3979
Signed-off-by: Yuqi Gu <yuqi.gu@arm.com>
Signed-off-by: Evans Ye <evansye@apache.org>
  • Loading branch information
guyuqi authored and evans-ye committed Feb 15, 2019
1 parent 48c3495 commit f1500c13124759897105ceb5270de5d242b2b5eb
Show file tree
Hide file tree
Showing 5 changed files with 61 additions and 54 deletions.
@@ -76,6 +76,11 @@

<% end %>
<% if @mapreduce_jobhistory_host %>
<property>
<name>mapred.job.tracker</name>
<value>BIGTOP-3137: Required by Giraph checkLocalJobRunnerConfiguration</value>
</property>

<property>
<name>mapreduce.jobhistory.address</name>
<value><%= @mapreduce_jobhistory_host %>:<%= @mapreduce_jobhistory_port %></value>
@@ -110,11 +110,12 @@ install -d -m 0755 ${PREFIX}/${LIB_DIR}
# Installing Giraph core
install -d -m 0755 ${PREFIX}/${LIB_DIR}
cp -r $BUILD_DIR/giraph-dist/target/giraph*-bin/*/* ${PREFIX}/${LIB_DIR}
cp $BUILD_DIR/giraph-examples/target/giraph-examples*.jar ${PREFIX}/${LIB_DIR}

# Installing docs and examples
install -d -m 0755 $PREFIX/${DOC_DIR}
# cp -r $BUILD_DIR/target/staging/* $PREFIX/${DOC_DIR}
mv ${PREFIX}/${LIB_DIR}/giraph-examples*.jar $PREFIX/${DOC_DIR}
cp ${PREFIX}/${LIB_DIR}/giraph-examples*.jar $PREFIX/${DOC_DIR}

# Install executable wrappers
install -d -m 0755 $PREFIX/usr/bin
@@ -134,6 +135,7 @@ for i in giraph ; do
# Workaround for GIRAPH-199
export HADOOP_HOME=\${HADOOP_HOME:-/usr/lib/hadoop}
export HADOOP_CONF_DIR=\${HADOOP_CONF_DIR:-/etc/hadoop/conf}
export HADOOP_CLASSPATH=\${HADOOP_CLASSPATH:-/etc/giraph/conf}
export GIRAPH_HOME=$LIB_DIR
exec $BIN_DIR/$i "\$@"
@@ -0,0 +1,37 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
def tests_to_include() {
return [
"TestGiraphSmoke.groovy"
];
}

sourceSets {
test {
groovy {
srcDirs = [
"${BIGTOP_HOME}/bigtop-tests/test-artifacts/giraph/src/main/groovy/org/apache/bigtop/itest/giraphsmoke/"
]
exclude { FileTreeElement elem -> (doExclude(elem.getName())) }
}
}
}

test.doFirst {
checkEnv(["GIRAPH_HOME"])
}
@@ -24,17 +24,12 @@ import org.apache.bigtop.itest.shell.Shell
import org.junit.Test

public class TestGiraphSmoke {
static String runnerScript = "HADOOP_CLASSPATH=/etc/giraph/conf hadoop jar"
static String testDir = "test.giraphsmoke." + (new Date().getTime())

static String giraphHome = System.getProperty('GIRAPH_HOME', '/usr/lib/giraph')
static String giraphJar = "${giraphHome}/giraph-jar-with-dependencies.jar"

static Shell sh = new Shell("/bin/bash -s")

@Test(timeout = 300000L)
public void testPageRankBenchmark() {
sh.exec("${runnerScript} ${giraphJar}"
sh.exec("hadoop jar /usr/lib/giraph/giraph-examples*with-dependencies.jar"
+ " org.apache.giraph.benchmark.PageRankBenchmark"
+ " -v " // verbose
+ " -e 1" // edges per vertex
@@ -47,7 +42,7 @@ public class TestGiraphSmoke {

@Test(timeout = 300000L)
public void testRandomMessageBenchmark() {
sh.exec("${runnerScript} ${giraphJar}"
sh.exec("hadoop jar /usr/lib/giraph/giraph-examples*with-dependencies.jar"
+ " org.apache.giraph.benchmark.RandomMessageBenchmark"
+ " -v " // verbose
+ " -e 1" // edges per vertex
@@ -61,55 +56,22 @@ public class TestGiraphSmoke {
}

@Test(timeout = 300000L)
public void testSimpleCheckpointVertex() {
sh.exec("hadoop fs -rmr ${testDir}");
sh.exec("${runnerScript} ${giraphJar}"
+ " org.apache.giraph.examples.SimpleCheckpointVertex"
+ " -v " // verbose
+ " -s 3" // number of supersteps
+ " -w 3" // workers
+ " -o ${testDir}"
)
assertEquals("running SimpleCheckpointVertex failed", 0, sh.getRet());
}

@Test(timeout = 300000L)
public void testSimpleVertexWithWorkerContext() {
sh.exec("hadoop fs -rmr ${testDir}");
sh.exec("${runnerScript} ${giraphJar}"
+ " org.apache.giraph.examples.SimpleVertexWithWorkerContext"
+ " ${testDir} 3"
)
assertEquals("running SimpleCheckpointVertex failed", 0, sh.getRet());
}

@Test(timeout = 300000L)
public void testSimpleShortestPathsVertex() {
// A graph definition:
// [vertex id, vertex value, [[edge1, value1], .. [edgeN, valueN]]]
List graphDescription = [[0, 0, [[1, 1], [2, 2]]],
[1, 1, [[2, 2], [3, 3]]],
[2, 2, [[3, 3], [4, 4]]],
[3, 3, [[4, 4], [5, 5]]],
[4, 4, [[5, 5], [0, 0]]],
[5, 5, [[0, 0], [1, 1]]]];
int partitionSize = 2;
public void testSimpleShortestPathsComputation() {
sh.exec("echo -e '[0,0,[[1,1],[3,3]]]\n[1,0,[[0,1],[2,2],[3,1]]]\n[2,0,[[1,2],[4,4]]]\n[3,0,[[0,3],[1,1],[4,4]]]\n[4,0,[[3,4],[2,4]]]' > ./giraphTest.txt");

sh.exec("hadoop fs -rmr ${testDir}",
"hadoop fs -mkdir ${testDir}/input");
sh.exec("hadoop fs -mkdir ${testDir}");

for (int i = 0; i < graphDescription.size(); i += partitionSize) {
String part = graphDescription[i..(i + partitionSize - 1)].join("\n");
int partId = i / partitionSize;
sh.exec("hadoop fs -put <(echo '${part}') ${testDir}/input/part-m-${partId}");
}
sh.exec("hadoop fs -copyFromLocal ./giraphTest.txt ${testDir}/giraphTest.txt");

sh.exec("${runnerScript} ${giraphJar}"
+ " org.apache.giraph.examples.SimpleShortestPathsVertex"
+ " ${testDir}/input"
+ " ${testDir}/output"
+ " 0 ${graphDescription.size() / partitionSize}"
sh.exec("hadoop jar /usr/lib/giraph/giraph-examples*with-dependencies.jar"
+ " org.apache.giraph.GiraphRunner"
+ " org.apache.giraph.examples.SimpleShortestPathsComputation"
+ " -vif org.apache.giraph.io.formats.JsonLongDoubleFloatDoubleVertexInputFormat"
+ " -vip ${testDir}/giraphTest.txt"
+ " -vof org.apache.giraph.io.formats.IdWithValueTextOutputFormat"
+ " -op ${testDir}/shortestpaths"
+ " -w 3"
)
assertEquals("running SimpleShortestPathsVertex failed", 0, sh.getRet());
assertEquals("running SimpleShortestPathsComputation failed", 0, sh.getRet());
}
}
@@ -47,6 +47,7 @@ export MAHOUT_HOME=${MAHOUT_HOME:-/usr/lib/mahout}
export SPARK_HOME=${SPARK_HOME:-/usr/lib/spark}
export SQOOP_HOME=${SQOOP_HOME:-/usr/lib/sqoop}
export ZOOKEEPER_HOME=${ZOOKEEPER_HOME:-/usr/lib/zookeeper}
export GIRAPH_HOME=${GIRAPH_HOME:-/usr/lib/giraph}

echo -e "\n===== START TO RUN SMOKE TESTS: $SMOKE_TESTS =====\n"

0 comments on commit f1500c1

Please sign in to comment.