Skip to content
Permalink
Browse files
BIGTOP-3151: Add flink smoke test
Add flink to smoke tests.

Change-Id: Ib6a538a97347ec0e77b51c81b09a9d23a1a25a1a
Signed-off-by: Jun He <jun.he@linaro.org>
  • Loading branch information
JunHe77 committed Feb 27, 2019
1 parent 44d62c7 commit 3134c0c0b883fbdf7d9254545f72722fe356c8f7
Show file tree
Hide file tree
Showing 5 changed files with 134 additions and 0 deletions.
@@ -0,0 +1,87 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.bigtop.itest.hadoop.flink

import org.apache.bigtop.itest.shell.Shell
import org.apache.commons.logging.Log
import org.apache.commons.logging.LogFactory
import org.junit.BeforeClass
import org.junit.AfterClass
import org.junit.Test

import static org.junit.Assert.assertEquals

class TestFlink {
static private Log LOG = LogFactory.getLog(Object.class)

static Shell sh = new Shell("/bin/bash -s")

static String cmdPrefix = "export HADOOP_CONF_DIR=/etc/hadoop/conf;HADOOP_USER_NAME=hdfs"
static private final String config_file = "/etc/flink/conf/flink-conf.yaml";


private static void execCommand(String cmd) {
LOG.info(cmd)

sh.exec("$cmdPrefix $cmd")
}

@BeforeClass
static void setupTest() {
execCommand("hadoop fs -rm -r /flink");
execCommand("hadoop fs -mkdir /flink")
execCommand("hadoop fs -put test.data /flink/")
}

@AfterClass
static void tearDown() {
execCommand("hadoop fs -rm -r /flink")
}

@Test
void testCheckRestfulAPI() {
// read JM address and port from conf
execCommand("awk '{if(/jobmanager.rpc.address:/) print \$2}' < "+ config_file);
final String jmhost = sh.out.join('\n');
execCommand("awk '{if(/jobmanager.web.port:/) print \$2}' < "+config_file);
final String webPort = sh.out.join('\n');
// check web API
execCommand("curl http://"+jmhost+":"+webPort+"/config");
final String result = sh.out.join('\n');
assert(result.contains("flink-version"));
}

@Test
void testWordCountBatch() {
execCommand("flink run \$FLINK_HOME/examples/batch/WordCount.jar --input hdfs:///flink/test.data --output hdfs:///tmp/result.txt")

execCommand("hadoop fs -cat /tmp/result.txt")

String expected =
"black 5\n" +
"blue 11\n" +
"green 11\n" +
"white 5\n" +
"yellow 11";

String actual = sh.out.join('\n')

assertEquals("Incorrect output", expected, actual)
}
}
@@ -0,0 +1,34 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
def tests_to_include() {
return [
"TestFlink.groovy"
];
}

sourceSets {
test {
groovy {
srcDirs = ["${BIGTOP_HOME}/bigtop-tests/smoke-tests/flink/"]
}
}
}

test.doFirst {
checkEnv(["FLINK_HOME"])
}
@@ -0,0 +1,11 @@
green blue yellow
yellow green blue black white
yellow green blue
white yellow green blue black
blue yellow green
black white yellow green blue
green blue yellow
blue black white yellow green
yellow green blue
green blue black white yellow
blue yellow green
@@ -74,6 +74,7 @@ rat {
/* Test data with rigid structure and/or binary */
"bigtop-tests/test-artifacts/**/resources/**",
"bigtop-tests/smoke-tests/alluxio/datafile",
"bigtop-tests/smoke-tests/flink/test.data",
"bigtop-tests/smoke-tests/hive/passwd.ql",
"bigtop-tests/smoke-tests/phoenix/*.csv",
"bigtop-tests/smoke-tests/phoenix/*.sql",
@@ -48,6 +48,7 @@ export SPARK_HOME=${SPARK_HOME:-/usr/lib/spark}
export SQOOP_HOME=${SQOOP_HOME:-/usr/lib/sqoop}
export ZOOKEEPER_HOME=${ZOOKEEPER_HOME:-/usr/lib/zookeeper}
export GIRAPH_HOME=${GIRAPH_HOME:-/usr/lib/giraph}
export FLINK_HOME=${FLINK_HOME:-/usr/lib/flink}

echo -e "\n===== START TO RUN SMOKE TESTS: $SMOKE_TESTS =====\n"

0 comments on commit 3134c0c

Please sign in to comment.