Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

FLUME-406: Update HBase Plugin Build to build and run hbase plugin tests

  • Loading branch information...
commit eef0f463c7c4b7c494ede440357459b19117d3c9 1 parent 0e98e9a
Dani Rayan authored jmhsieh committed
View
123 plugins/hbasesink/build.xml
@@ -25,11 +25,36 @@
<project name="flume-hbase" default="jar">
<property name="javac.debug" value="on"/>
+ <property name="javac.optimize" value="on"/>
+ <property name="javac.deprecation" value="off"/>
+ <property name="javac.version" value="1.5"/>
+ <property name="javac.args" value=""/>
+ <property name="javac.args.warnings" value="-Xlint:unchecked"/>
+
<property name="flume.base" value="../.."/>
+ <property name="build.dir" value="build"/>
+ <property name="build.test" value="${build.dir}/test"/>
+ <property name="build.encoding" value="ISO-8859-1"/>
+
+ <property name="test.src.dir" value="src/javatest"/>
+ <property name="test.lib.dir" value="${flume.base}/libtest"/>
+ <property name="test.build.dir" value="${build.dir}/test"/>
+ <property name="test.generated.dir" value="${test.build.dir}/src"/>
+ <property name="test.build.data" value="${test.build.dir}/data"/>
+ <property name="test.log.dir" value="${test.build.dir}/logs"/>
+ <property name="test.build.classes" value="${test.build.dir}/classes"/>
+ <property name="test.include" value="Test*"/>
+ <property name="test.classpath.id" value="test.classpath"/>
+ <property name="test.output" value="yes"/>
+ <property name="test.timeout" value="900000"/>
+ <property name="test.junit.output.format" value="plain"/>
<path id="classpath">
<!-- in case we are running in dev env -->
<pathelement location="${flume.base}/build/classes"/>
+ <fileset dir="${flume.base}">
+ <include name="flume-*.jar" />
+ </fileset>
<fileset dir="${flume.base}/lib">
<include name="**/slf4j-*.jar" />
<include name="**/hadoop-*.jar" />
@@ -38,21 +63,40 @@
<fileset dir="${flume.base}/plugins/hbasesink/lib">
<include name="**/*.jar" />
</fileset>
- <!-- in case we are running in release env -->
- <fileset dir="${flume.base}">
- <include name="flume-*.jar" />
+ </path>
+
+ <!-- the unit test classpath: uses test.src.dir for configuration -->
+ <path id="test.classpath">
+ <pathelement location="${test.build.classes}" />
+ <pathelement location="${test.src.dir}"/>
+ <pathelement location="${build.dir}"/>
+ <fileset dir="${test.lib.dir}">
+ <include name="**/*.jar" />
+ <exclude name="**/excluded/" />
</fileset>
+ <fileset dir="${flume.base}/lib">
+ <include name="**/*.jar" />
+ <exclude name="**/excluded/" />
+ </fileset>
+ <fileset dir="${flume.base}/plugins/hbasesink">
+ <include name="**/*.jar" />
+ <exclude name="**/excluded/" />
+ </fileset>
+ <fileset dir="${flume.base}/plugins/hbasesink/lib">
+ <include name="**/*.jar" />
+ </fileset>
+ <path refid="classpath"/>
</path>
<target name="jar">
- <mkdir dir="build"/>
- <mkdir dir="build/classes"/>
+ <mkdir dir="${build.dir}"/>
+ <mkdir dir="${build.dir}/classes"/>
- <javac srcdir="./src/java" destdir="build/classes" debug="${javac.debug}">
+ <javac srcdir="./src/java" destdir="${build.dir}/classes" debug="${javac.debug}">
<classpath refid="classpath"/>
</javac>
- <jar jarfile="hbase_sink.jar" basedir="build/classes"/>
+ <jar jarfile="hbase_sink.jar" basedir="${build.dir}/classes"/>
</target>
<target name="clean">
@@ -61,4 +105,69 @@
<delete file="hbase_sink.jar" />
</target>
+ <!-- ================================================================== -->
+ <!-- Compile test code -->
+ <!-- ================================================================== -->
+ <target name="compile-plugin-test" depends="jar">
+ <mkdir dir="${build.dir}/test"/>
+ <mkdir dir="${build.dir}/test/classes"/>
+
+ <javac
+ encoding="${build.encoding}"
+ srcdir="${test.src.dir}"
+ includes="**/*.java"
+ destdir="${test.build.classes}"
+ debug="${javac.debug}"
+ optimize="${javac.optimize}"
+ target="${javac.version}"
+ source="${javac.version}"
+ deprecation="${javac.deprecation}">
+ <compilerarg line="${javac.args}" />
+ <classpath refid="test.classpath"/>
+ </javac>
+ </target>
+
+
+ <!-- ================================================================== -->
+ <!-- Run unit tests -->
+ <!-- ================================================================== -->
+ <target name="test" depends="jar,compile-plugin-test" >
+ <echo message="Unit Testing of HBase Sink"/>
+ <junit
+ printsummary="yes" showoutput="${test.output}"
+ haltonfailure="no" fork="yes" maxmemory="1024m"
+ errorProperty="tests.failed" failureProperty="tests.failed"
+ timeout="${test.timeout}"
+ dir="${test.build.dir}">
+
+ <!-- uncomment this if you want to attach a debugger -->
+ <!--
+ <jvmarg line="-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=2601" />
+ -->
+ <env key="FLUME_HOME" value="${basedir}" />
+ <sysproperty key="javax.xml.parsers.DocumentBuilderFactory" value="com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl" />
+ <sysproperty key="java.library.path" value="${lib.dir}" />
+ <sysproperty key="build.test" value="${build.test}"/>
+
+ <!-- we want more log4j output when running unit tests -->
+ <sysproperty key="hadoop.root.logger"
+ value="INFO,console" />
+
+ <!-- tools.jar from Sun JDK also required to invoke javac. -->
+ <classpath>
+ <path refid="test.classpath"/>
+ <pathelement path="${env.JAVA_HOME}/lib/tools.jar" />
+ </classpath>
+ <formatter type="${test.junit.output.format}" />
+ <batchtest todir="${build.test}" unless="testcase">
+ <fileset dir="${test.src.dir}"
+ includes="**/Test*.java" excludes="**/${test.exclude}.java" />
+ </batchtest>
+ <batchtest todir="${build.test}" if="testcase">
+ <fileset dir="${test.src.dir}" includes="**/${testcase}.java"/>
+ </batchtest>
+ </junit>
+ <fail if="tests.failed">Tests failed!</fail>
+ </target>
+
</project>
View
36 plugins/hbasesink/src/javatest/com/cloudera/flume/hbase/TestAttr2HBaseSink.java
@@ -68,7 +68,9 @@ public void testSink() throws IOException {
final String tableSysFamily = "sysFamily";
final String tableFamily1 = "family1";
final String tableFamily2 = "family2";
- final String tableBodyMap = "sysFamily:event";
+ final String tableBody = "sysFamily";
+ final String tableBodyFamily = "event";
+ final Boolean writeBody = true;
// create the table and column family to be used by sink
HTableDescriptor desc = new HTableDescriptor(tableName);
@@ -79,7 +81,7 @@ public void testSink() throws IOException {
admin.createTable(desc);
// explicit constructor rather than builder - we want to control the conf
- EventSink snk = new Attr2HBaseEventSink(tableName, tableSysFamily, tableBodyMap, "2hb_", 0, true, hbaseEnv.conf);
+ EventSink snk = new Attr2HBaseEventSink(tableName, tableSysFamily, writeBody, tableBody, tableBodyFamily, "2hb_", 0, true, hbaseEnv.conf);
snk.open();
try {
Event e1 = new EventImpl("message0".getBytes(), Clock.unixTime(),
@@ -139,7 +141,10 @@ public void testSink() throws IOException {
@Test
public void testCreatePutWithoutSystemColumnFamily() {
- Attr2HBaseEventSink snk = new Attr2HBaseEventSink("tableName", "", true, "2hb_", 0, true, null);
+ final String tableBody = "sysFamily";
+ final String tableBodyFamily = "event";
+ final Boolean writeBody = true;
+ Attr2HBaseEventSink snk = new Attr2HBaseEventSink("tableName", "", writeBody, tableBody, tableBodyFamily, "2hb_", 0, true, null);
Event e = new EventImpl("message".getBytes(), Clock.unixTime(),
Priority.INFO, 0, "localhost");
@@ -157,7 +162,10 @@ public void testCreatePutWithoutSystemColumnFamily() {
@Test
public void testDontWriteBody() {
- Attr2HBaseEventSink snk = new Attr2HBaseEventSink("tableName", "sysFam", false, "2hb_", 0, true, null);
+ final String tableBody = "";
+ final String tableBodyFamily = "";
+ final Boolean writeBody = false;
+ Attr2HBaseEventSink snk = new Attr2HBaseEventSink("tableName", "sysFam", writeBody, tableBody, tableBodyFamily, "2hb_", 0, true, null);
Event e = new EventImpl("message".getBytes(), Clock.unixTime(),
Priority.INFO, 0, "localhost");
@@ -171,7 +179,10 @@ public void testDontWriteBody() {
@Test
public void testCreatePutWithoutExplicitRowKey() {
- Attr2HBaseEventSink snk = new Attr2HBaseEventSink("tableName", "sysFam", true, "2hb_", 0, true, null);
+ final String tableBody = "sysFamily";
+ final String tableBodyFamily = "event";
+ final Boolean writeBody = true;
+ Attr2HBaseEventSink snk = new Attr2HBaseEventSink("tableName", "sysFam", writeBody, tableBody, tableBodyFamily, "2hb_", 0, true, null);
Event e = new EventImpl("message".getBytes(), Clock.unixTime(),
Priority.INFO, 0, "localhost");
@@ -183,7 +194,10 @@ public void testCreatePutWithoutExplicitRowKey() {
@Test
public void testAddAttributeWithSystemColumnFamSpecified() {
- Attr2HBaseEventSink snk = new Attr2HBaseEventSink("tableName", "sysFam", true, "2hb_", 0, true, null);
+ final String tableBody = "sysFamily";
+ final String tableBodyFamily = "event";
+ final Boolean writeBody = true;
+ Attr2HBaseEventSink snk = new Attr2HBaseEventSink("tableName", "sysFam",writeBody, tableBody, tableBodyFamily, "2hb_", 0, true, null);
byte[] foo = Bytes.toBytes("foo");
Put put = new Put(foo);
@@ -203,7 +217,10 @@ public void testAddAttributeWithSystemColumnFamSpecified() {
@Test
public void testAddAttributeWithoutSystemColumnFam() {
- Attr2HBaseEventSink snk = new Attr2HBaseEventSink("tableName", "", true, "2hb_", 0, true, null);
+ final String tableBody = "sysFamily";
+ final String tableBodyFamily = "event";
+ final Boolean writeBody = true;
+ Attr2HBaseEventSink snk = new Attr2HBaseEventSink("tableName", "", writeBody, tableBody, tableBodyFamily, "2hb_", 0, true, null);
byte[] foo = Bytes.toBytes("foo");
Put put = new Put(foo);
@@ -236,7 +253,10 @@ private void assertEmpty(Put put) {
@Test
public void testAddAttributeWithSystemColumnFamSpecifiedAndEmptyPrefix() {
- Attr2HBaseEventSink snk = new Attr2HBaseEventSink("tableName", "sysFam", true, "", 0, true, null);
+ final String tableBody = "sysFamily";
+ final String tableBodyFamily = "event";
+ final Boolean writeBody = true;
+ Attr2HBaseEventSink snk = new Attr2HBaseEventSink("tableName", "sysFam", writeBody, tableBody, tableBodyFamily, "", 0, true, null);
byte[] foo = Bytes.toBytes("foo");
Put put = new Put(foo);
View
121 plugins/hbasesink/src/javatest/com/cloudera/flume/hbase/TestHBaseSink.java
@@ -1,121 +0,0 @@
-// This Cloudera, Inc. source code, including without limit any
-// human-readable computer programming code and associated documentation
-// (together "Source Code"), contains valuable confidential, proprietary
-// and trade secret information of Cloudera and is protected by the laws
-// of the U.S. and other countries. Any use of the Source Code, including
-// without limit any disclosure or reproduction, without the prior
-// written authorization of Cloudera is strictly prohibited.
-//
-// Copyright (c) 2010 Cloudera, Inc. All rights reserved.
-package com.cloudera.flume.hbase;
-
-import java.io.IOException;
-
-import junit.framework.Assert;
-
-import org.apache.hadoop.hbase.HBaseTestCase;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import com.cloudera.flume.core.Event;
-import com.cloudera.flume.core.EventImpl;
-import com.cloudera.flume.core.EventSink;
-import com.cloudera.flume.core.Event.Priority;
-import com.cloudera.util.Clock;
-
-/**
- * Test the hbase sink writes events to a table/family properly
- */
-public class TestHBaseSink {
- private static HBaseTestEnv hbaseEnv;
-
- @BeforeClass
- public static void setup() throws Exception {
- // expensive, so just do it once for all tests, just make sure
- // that tests don't overlap (use diff tables for each test)
- hbaseEnv = new HBaseTestEnv();
- hbaseEnv.conf.set(HBaseTestCase.TEST_DIRECTORY_KEY, "build/test/data");
- hbaseEnv.setUp();
- }
-
- @AfterClass
- public static void teardown() throws Exception {
- hbaseEnv.tearDown();
- }
-
- /**
- * Write events to a sink directly, verify by scanning HBase table.
- */
- @Test
- public void testSink() throws IOException {
- final String tableName = "testSink";
- final String tableFamily = "testSinkFamily";
-
- // create the table and column family to be used by sink
- HTableDescriptor desc = new HTableDescriptor(tableName);
- desc.addFamily(new HColumnDescriptor(tableFamily));
- HBaseAdmin admin = new HBaseAdmin(hbaseEnv.conf);
- admin.createTable(desc);
-
- // explicit constructor rather than builder - we want to control the conf
- EventSink snk = new HBaseEventSink(tableName, tableFamily, hbaseEnv.conf);
- snk.open();
- try {
- long day_millis = 1000 * 60 * 60 * 24;
- Event e1 = new EventImpl("message0".getBytes(), Clock.unixTime(),
- Priority.INFO, 0, "localhost");
- Event e2 = new EventImpl("message1".getBytes(),
- e1.getTimestamp() + day_millis, Priority.INFO, 1, "localhost");
- Event e3 = new EventImpl("message2".getBytes(),
- e1.getTimestamp() + 2 * day_millis, Priority.INFO, 2, "localhost");
- snk.append(e1);
- snk.append(e2);
- snk.append(e3);
- } finally {
- snk.close();
- }
-
- // verify that the events made it into hbase
- HTable table = new HTable(hbaseEnv.conf, tableName);
- try {
- for(long i = 0; i <=2; i++) {
- Result r = table.get(new Get(Bytes.toBytes(i)));
- System.out.println("result " + r);
- byte [] value = r.getValue(Bytes.toBytes(tableFamily),
- Bytes.toBytes("event"));
- Assert.assertEquals("Matching body", "message" + i, Bytes.toString(value));
- }
- } finally {
- table.close();
- }
- }
-
- /** Useful for debugging purposes. Scan table and dump to stdout */
- private void dumpByTimestamp(HTable table, String tableFamily, String colName)
- throws IOException
- {
- Scan s = new Scan();
- s.addColumn(Bytes.toBytes(tableFamily), Bytes.toBytes(colName));
- ResultScanner scanner = table.getScanner(s);
- System.out.println("Scanning " + table.toString() + " " + tableFamily);
- try {
- for (Result rr = scanner.next(); rr != null; rr = scanner.next()) {
- System.out.println("Found row: " + rr);
- }
-
- } finally {
- scanner.close();
- }
- }
-
-}
Please sign in to comment.
Something went wrong with that request. Please try again.