Permalink
Browse files

HIVE-705. Read HBase tables via Hive

(John Sichi via namit)



git-svn-id: https://svn.apache.org/repos/asf/hadoop/hive/trunk@922404 13f79535-47bb-0310-9956-ffa450edef68
  • Loading branch information...
1 parent 90d8b77 commit 29be62ce99ad7894c868ea1bac1737b174356732 Namit Jain committed Mar 12, 2010
Showing with 4,212 additions and 146 deletions.
  1. +5 −2 CHANGES.txt
  2. +1 −0 build-common.xml
  3. +13 −2 build.xml
  4. +2 −0 hbase-handler/README.txt
  5. +97 −0 hbase-handler/build.xml
  6. +8 −0 hbase-handler/ivy.xml
  7. BIN hbase-handler/lib/hbase-0.20.3-test.jar
  8. BIN hbase-handler/lib/hbase-0.20.3.jar
  9. BIN hbase-handler/lib/zookeeper-3.2.2.jar
  10. +495 −0 hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSerDe.java
  11. +85 −0 hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSplit.java
  12. +255 −0 hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStorageHandler.java
  13. +179 −0 hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java
  14. +87 −0 hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableOutputFormat.java
  15. +154 −0 hbase-handler/src/java/org/apache/hadoop/hive/hbase/LazyHBaseCellMap.java
  16. +175 −0 hbase-handler/src/java/org/apache/hadoop/hive/hbase/LazyHBaseRow.java
  17. +21 −0 hbase-handler/src/java/org/apache/hadoop/hive/hbase/package-info.java
  18. +103 −0 hbase-handler/src/test/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java
  19. +135 −0 hbase-handler/src/test/org/apache/hadoop/hive/hbase/TestHBaseSerDe.java
  20. +292 −0 hbase-handler/src/test/org/apache/hadoop/hive/hbase/TestLazyHBaseObject.java
  21. +140 −0 hbase-handler/src/test/queries/hbase_queries.q
  22. +889 −0 hbase-handler/src/test/results/hbase_queries.q.out
  23. +123 −0 hbase-handler/src/test/templates/TestHBaseCliDriver.vm
  24. +1 −1 metastore/if/hive_metastore.thrift
  25. +2 −0 metastore/src/gen-cpp/hive_metastore_constants.cpp
  26. +1 −0 metastore/src/gen-cpp/hive_metastore_constants.h
  27. +2 −0 metastore/src/gen-javabean/org/apache/hadoop/hive/metastore/api/Constants.java
  28. +2 −0 metastore/src/gen-php/hive_metastore_constants.php
  29. +2 −0 metastore/src/gen-py/hive_metastore/constants.py
  30. +92 −0 metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaHook.java
  31. +39 −0 metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaHookLoader.java
  32. +3 −11 metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
  33. +50 −2 metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
  34. +39 −12 metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
  35. +1 −1 metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
  36. +45 −17 ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
  37. +11 −4 ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java
  38. +28 −17 ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java
  39. +29 −12 ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
  40. +8 −0 ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java
  41. +18 −0 ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
  42. +8 −0 ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
  43. +80 −0 ql/src/java/org/apache/hadoop/hive/ql/metadata/DefaultStorageHandler.java
  44. +31 −2 ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
  45. +82 −0 ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java
  46. +23 −0 ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveUtils.java
  47. +61 −6 ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java
  48. +20 −0 ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
  49. +13 −9 ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
  50. +3 −1 ql/src/java/org/apache/hadoop/hive/ql/parse/ErrorMsg.java
  51. +4 −0 ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g
  52. +3 −0 ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java
  53. +48 −28 ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
  54. +19 −1 ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableDesc.java
  55. +12 −0 ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableDesc.java
  56. +22 −15 ql/src/java/org/apache/hadoop/hive/ql/plan/MapredLocalWork.java
  57. +3 −1 ql/src/java/org/apache/hadoop/hive/ql/plan/PartitionDesc.java
  58. +41 −0 ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java
  59. +21 −0 ql/src/java/org/apache/hadoop/hive/ql/plan/TableDesc.java
  60. +8 −1 ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java
  61. +6 −0 ql/src/test/queries/clientnegative/alter_non_native.q
  62. +5 −0 ql/src/test/queries/clientnegative/load_non_native.q
  63. +16 −0 ql/src/test/results/clientnegative/alter_non_native.q.out
  64. +12 −0 ql/src/test/results/clientnegative/load_non_native.q.out
  65. +8 −1 serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyMap.java
  66. +8 −0 serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyObject.java
  67. +23 −0 serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyStruct.java
View
@@ -48,8 +48,8 @@ Trunk - Unreleased
HIVE-1194. Add sort merge join
(He Yongqiang via namit)
- HIVE-1241. Drop the table at the beginning of tests
- (He Yongqiang via namit)
+ HIVE-705. Read HBase tables via Hive
+ (John Sichi via namit)
IMPROVEMENTS
HIVE-983. Function from_unixtime takes long.
@@ -258,6 +258,9 @@ Trunk - Unreleased
HIVE-1022. desc Table should work.
(namit via He Yongqiang)
+ HIVE-1241. Drop the table at the beginning of tests
+ (He Yongqiang via namit)
+
Release 0.5.0 - Unreleased
INCOMPATIBLE CHANGES
View
@@ -429,6 +429,7 @@
<target name="clean-test">
<delete dir="${test.build.dir}"/>
+ <delete dir="${build.dir.hive}/ql/tmp"/>
<delete dir="${build.dir.hive}/test"/>
</target>
View
@@ -46,6 +46,16 @@
<property name="checkstyle.conf.dir" location="${hive.root}/checkstyle"/>
<property name="checkstyle.build.dir" location="${build.dir.hive}/checkstyle"/>
+ <!-- hbase-handler is enabled by default only for Hadoop 0.20.x -->
+ <condition property="hbase.enabled" value="true" else="false">
+ <matches string="${hadoop.version.ant-internal}" pattern="^0\.20\..*" />
+ </condition>
+
+ <!-- only iterate over hbase-handler when enabled -->
+ <condition property="hbase.iterate" value=",hbase-handler/build.xml" else="">
+ <istrue value="${hbase.enabled}"/>
+ </condition>
+
<condition property="is-offline" value="true" else="false">
<isset property="offline"/>
</condition>
@@ -82,7 +92,7 @@
<subant target="@{target}">
<property name="build.dir.hive" location="${build.dir.hive}"/>
<property name="is-offline" value="${is-offline}"/>
- <filelist dir="." files="ant/build.xml,shims/build.xml,common/build.xml,serde/build.xml,metastore/build.xml,ql/build.xml,cli/build.xml,contrib/build.xml,service/build.xml,jdbc/build.xml,hwi/build.xml,ant/build.xml"/>
+ <filelist dir="." files="ant/build.xml,shims/build.xml,common/build.xml,serde/build.xml,metastore/build.xml,ql/build.xml,cli/build.xml,contrib/build.xml,service/build.xml,jdbc/build.xml,hwi/build.xml${hbase.iterate},ant/build.xml"/>
</subant>
</sequential>
</macrodef>
@@ -93,7 +103,7 @@
<subant target="@{target}">
<property name="build.dir.hive" location="${build.dir.hive}"/>
<property name="is-offline" value="${is-offline}"/>
- <filelist dir="." files="shims/build.xml,common/build.xml,serde/build.xml,metastore/build.xml,ql/build.xml,cli/build.xml,contrib/build.xml,service/build.xml,jdbc/build.xml,hwi/build.xml"/>
+ <filelist dir="." files="shims/build.xml,common/build.xml,serde/build.xml,metastore/build.xml,ql/build.xml,cli/build.xml,contrib/build.xml,service/build.xml,jdbc/build.xml,hwi/build.xml${hbase.iterate}"/>
</subant>
</sequential>
</macrodef>
@@ -286,6 +296,7 @@
<fileset file="${build.dir.hive}/metastore/hive-metastore-${version}.jar"/>
<fileset file="${build.dir.hive}/hwi/hive-hwi-${version}.war"/>
<fileset file="${build.dir.hive}/contrib/hive-contrib-${version}.jar"/>
+ <fileset file="${build.dir.hive}/contrib/hive-hbase_handler-${version}.jar"/>
</copy>
<copy todir="${target.example.dir}/files" preservelastmodified="true" flatten="true">
<fileset dir="${test.data.dir}/files" includes="*.*" excludes="**/.svn"/>
View
@@ -0,0 +1,2 @@
+See http://wiki.apache.org/hadoop/Hive/HBaseIntegration for
+information about the HBase storage handler.
View
@@ -0,0 +1,97 @@
+<?xml version="1.0"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+
+<project name="hbase-handler" default="jar">
+
+ <property name="hbase-handler.lib.dir" value="${basedir}/lib"/>
+ <property name="src.dir" location="${basedir}/src/java"/>
+ <property name="hbase-handler.test.query.dir" location="${basedir}/src/test/queries"/>
+ <property name="ql.test.template.dir" location="${basedir}/../ql/src/test/templates"/>
+ <property name="ql.hbase.test.template.dir" location="${basedir}/src/test/templates"/>
+ <property name="hbase-handler.test.results.dir" location="${basedir}/src/test/results"/>
+
+ <import file="../build-common.xml"/>
+
+ <path id="test.classpath">
+ <pathelement location="${build.dir.hive}/ql/test/classes" />
+ <pathelement location="${test.build.classes}" />
+ <pathelement location="" />
+ <pathelement location="${hadoop.conf.dir}"/>
+ <pathelement location="${test.data.dir}/conf"/>
+ <pathelement location="${hive.conf.dir}"/>
+ <pathelement location="${hive.root}/cli/lib/jline-0.9.94.jar"/>
+ <pathelement location="${hadoop.test.jar}"/>
+ <pathelement location="${jetty.test.jar}"/>
+ <pathelement location="${servlet.test.jar}"/>
+ <pathelement location="${jasper.test.jar}"/>
+ <pathelement location="${jasperc.test.jar}"/>
+ <pathelement location="${jsp.test.jar}"/>
+ <pathelement location="${common.jar}"/>
+ <fileset dir="${hive.root}" includes="testlibs/*.jar"/>
+ <fileset dir="${hadoop.root}/lib" includes="*.jar"/>
+ <fileset dir="${hadoop.root}/lib/jsp-2.1" includes="*.jar"/>
+ <path refid="classpath"/>
+ </path>
+
+ <target name="test-jar" depends="compile-test, jar">
+ <delete file="${test.build.dir}/test-udfs.jar"/>
+ <jar jarfile="${test.build.dir}/test-udfs.jar">
+ <fileset dir="${test.build.classes}" includes="**/udf/*.class"/>
+ <fileset dir="${test.build.classes}" includes="**/udf/generic/*.class"/>
+ </jar>
+ </target>
+
+ <target name="gen-test" depends="deploy-ant-tasks, test-conditions, test-init" >
+ <taskdef name="qtestgen" classname="org.apache.hadoop.hive.ant.QTestGenTask"
+ classpath="${build.dir.hive}/anttasks/hive-anttasks-${version}.jar:${hive.root}/lib/velocity-1.5.jar:${hive.root}/lib/commons-collections-3.2.1.jar:${hive.root}/lib/commons-lang-2.4.jar"/>
+
+ <mkdir dir="${test.build.src}/org/apache/hadoop/hive/cli"/>
+ <mkdir dir="${test.log.dir}/hbase-handler"/>
+ <mkdir dir="${hbase-handler.test.results.dir}"/>
+
+ <qtestgen outputDirectory="${test.build.src}/org/apache/hadoop/hive/cli"
+ templatePath="${ql.hbase.test.template.dir}" template="TestHBaseCliDriver.vm"
+ queryDirectory="${hbase-handler.test.query.dir}"
+ queryFile="${qfile}"
+ clusterMode="${clustermode}"
+ resultsDirectory="${hbase-handler.test.results.dir}" className="TestHBaseCliDriver"
+ logFile="${test.log.dir}/testhbaseclidrivergen.log"
+ logDirectory="${test.log.dir}/hbase-handler"/>
+
+ </target>
+
+ <!-- override target jar because tests need to add hbase-handler jars,
+ which needs to be a constant without the version number -->
+ <target name="jar" depends="compile">
+ <echo message="Jar: ${ant.project.name}"/>
+ <jar
+ jarfile="${build.dir}/hive_${ant.project.name}.jar"
+ basedir="${build.classes}">
+ <manifest>
+ <!-- Not putting these in their own manifest section, since that inserts
+ a new-line, which breaks the reading of the attributes. -->
+ <attribute name="Implementation-Title" value="Hive"/>
+ <attribute name="Implementation-Version" value="${version}"/>
+ <attribute name="Implementation-Vendor" value="Apache"/>
+ </manifest>
+ </jar>
+ </target>
+
+</project>
View
@@ -0,0 +1,8 @@
+<ivy-module version="2.0">
+ <info organisation="org.apache.hadoop.hive" module="contrib"/>
+ <dependencies>
+ <dependency org="hadoop" name="core" rev="${hadoop.version.ant-internal}">
+ <artifact name="hadoop" type="source" ext="tar.gz"/>
+ </dependency>
+ </dependencies>
+</ivy-module>
Binary file not shown.
Binary file not shown.
Binary file not shown.
Oops, something went wrong.

0 comments on commit 29be62c

Please sign in to comment.