Merge remote-tracking branch 'apache/trunk' into trunk

apache · Apr 26, 2012 · 11a9e64 · 11a9e64
2 parents 956bc09 + 4edf979
commit 11a9e64
Show file tree

Hide file tree

Showing 44 changed files with 1,338 additions and 414 deletions.
diff --git a/build-common.xml b/build-common.xml
@@ -66,20 +66,38 @@
   <property name="test.print.classpath" value="false"/>
   <property name="test.lang" value="en_US.UTF-8"/>
 
+  <property name="hadoop.opts.23" value="-D mapreduce.framework.name=local" />
+  <property name="hadoop.opts.20" value="" />
+
   <path id="test.classpath">
     <pathelement location="${test.build.classes}" />
     <pathelement location="${test.build.resources}" />
     <pathelement location="" />
     <pathelement location="${test.src.data.dir}/conf"/>
     <pathelement location="${hive.conf.dir}"/>
-    <fileset dir="${hive.root}" includes="testlibs/*.jar"/>
-    <path refid="classpath"/>
-    <fileset dir="${hadoop.root}">
-      <!-- below is for 0.23 onwards -->
-      <include name="share/hadoop/common/lib/*.jar" />
-      <exclude name="share/hadoop/common/lib/hadoop-mapreduce-*.jar" />
-      <exclude name="share/hadoop/common/lib/hadoop-yarn-*.jar" />
-    </fileset>
+    <pathelement location="${build.dir.hive}/cli/test/classes"/>
+    <pathelement location="${build.dir.hive}/common/test/classes"/>
+    <pathelement location="${build.dir.hive}/hbase-handler/test/classes"/>
+    <pathelement location="${build.dir.hive}/hwi/test/classes"/>
+    <pathelement location="${build.dir.hive}/jdbc/test/classes"/>
+    <pathelement location="${build.dir.hive}/metastore/test/classes"/>
+    <pathelement location="${build.dir.hive}/ql/test/classes"/>
+    <pathelement location="${build.dir.hive}/serde/test/classes"/>
+    <pathelement location="${build.dir.hive}/service/test/classes"/>
+    <pathelement location="${build.dir.hive}/shims/test/classes"/>
+
+    <fileset dir="${hive.root}/build/ivy/lib/test" includes="*.jar" erroronmissingdir="false" excludes="**/hive_contrib*.jar,**/hive-contrib*.jar"/>
+    <fileset dir="${hive.root}/build/ivy/lib/default" includes="*.jar" erroronmissingdir="false" excludes="**/hive_contrib*.jar,**/hive-contrib*.jar" />
+    <fileset dir="${hive.root}/testlibs" includes="*.jar"/>
+    <fileset dir="${hive.root}/build/ivy/lib/hadoop0.${hadoop.mr.rev}.shim" includes="*.jar" />
+    <pathelement location="${build.classes}" />
+  </path>
+
+  <!-- include contrib on local classpath, but not on cluster -->
+  <!-- https://reviews.facebook.net/D2133#comment-47 -->
+  <path id="test.local.classpath">
+    <path refid="${test.classpath.id}"/>
+    <fileset dir="${hive.root}/build/ivy/lib/test" includes="hive-contrib*.jar" erroronmissingdir="false"/>
   </path>
 
 
@@ -125,47 +143,41 @@
   	<ivy:retrieve settingsRef="${ant.project.name}-${hadoop.version.ant-internal}.ivy.settings"
       pattern="${build.dir.hadoop}/[artifact]-[revision].[ext]"/>
   </target>
-
-  <available property="hadoopcore.${hadoop.version.ant-internal}.install.done"
-    file="${build.dir.hadoop}/hadoop-${hadoop.version.ant-internal}.installed"/>
 
-  <target name="install-hadoopcore-internal" depends="ivy-retrieve-hadoop-source"
-    unless="hadoopcore.${hadoop.version.ant-internal}.install.done">
+  <target name="ivy-resolve-test" depends="ivy-init-settings" unless="offline">
     <echo message="Project: ${ant.project.name}"/>
-    <untar src="${build.dir.hadoop}/hadoop-${hadoop.version.ant-internal}.tar.gz" dest="${build.dir.hadoop}" compression="gzip"/>
-    <chmod file="${hadoop.root}/bin/hadoop" perm="+x"/>
-    <touch file="${build.dir.hadoop}/hadoop-${hadoop.version.ant-internal}.installed"/>
+    <ivy:resolve settingsRef="${ant.project.name}.ivy.settings"
+      conf="test" log="${ivyresolvelog}"/>
   </target>
 
-  <target name="install-hadoopcore-default" unless="hadoop.root.nondefault">
+  <target name="ivy-retrieve-test" depends="ivy-resolve-test"
+    description="Retrieve Ivy-managed artifacts">
     <echo message="Project: ${ant.project.name}"/>
-    <antcall target="install-hadoopcore-internal"/>
+    <ivy:retrieve settingsRef="${ant.project.name}.ivy.settings"
+      pattern="${build.ivy.lib.dir}/${ivy.artifact.retrieve.pattern}"
+      log="${ivyresolvelog}" conf="test"/>
   </target>
 
-  <target name="install-hadoopcore">
+  <target name="ivy-resolve-hadoop-shim" depends="ivy-init-settings" unless="offline">
     <echo message="Project: ${ant.project.name}"/>
-    <condition property="hadoop.root.nondefault">
-      <not>
-        <equals arg1="${hadoop.root}" arg2="${hadoop.root.default}"/>
-      </not>
-    </condition>
-    <antcall target="install-hadoopcore-default"/>
+    <ivy:resolve settingsRef="${ant.project.name}.ivy.settings"
+      conf="${ivy.hadoop.shim.conf}" log="${ivyresolvelog}"/>
+  </target>
+
+  <target name="ivy-retrieve-hadoop-shim" depends="ivy-resolve-hadoop-shim"
+    description="Retrieve Ivy-managed artifacts">
+    <echo message="Project: ${ant.project.name}"/>
+    <ivy:retrieve settingsRef="${ant.project.name}.ivy.settings"
+      pattern="${build.ivy.lib.dir}/${ivy.artifact.retrieve.pattern}"
+      log="${ivyresolvelog}" conf="${ivy.hadoop.shim.conf}"/>
   </target>
 
   <!-- the normal classpath -->
   <path id="common-classpath">
-    <pathelement location="${hadoop.oldstyle-name.jar}"/>
-    <pathelement location="${hadoop.oldstyle-name.tools.jar}"/>
-    <pathelement location="${hadoop.newstyle-name.jar}"/>
-    <pathelement location="${hadoop.newstyle-name.tools.jar}"/>
-    <pathelement location="${hadoop.common.jar}"/>
-    <pathelement location="${hadoop.hdfs.jar}"/>
-    <pathelement location="${hadoop.mapreduce.jar}"/>
-    <pathelement location="${hadoop.mapreduce.tools.jar}"/>
     <pathelement location="${build.dir.hive}/classes"/>
     <fileset dir="${build.dir.hive}" includes="*/*.jar"/>
     <fileset dir="${hive.root}/lib" includes="*.jar"/>
-    <fileset dir="${build.ivy.lib.dir}/default" includes="*.jar" excludes="*hadoop*.jar"
+    <fileset dir="${build.ivy.lib.dir}/default" includes="*.jar" 
              erroronmissingdir="false"/>
   </path>
 
@@ -178,10 +190,10 @@
     <pathelement location="${build.dir.hive}/cli/classes"/>
     <pathelement location="${build.dir.hive}/shims/classes"/>
     <pathelement location="${build.dir.hive}/hwi/classes"/>
-    <fileset erroronmissingdir="false" dir="${build.dir.hadoop}/hadoop-${hadoop.security.version}"
-             includes="lib/commons-codec-*.jar"/>
-    <path refid="common-classpath"/>
+    <pathelement location="${build.dir.hive}/jdbc/classes"/>
+    <pathelement location="${build.dir.hive}/hbase-handler/classes"/>
     <fileset dir="${basedir}" includes="lib/*.jar"/>
+    <path refid="common-classpath"/>
   </path>
 
   <target name="create-dirs">
@@ -251,10 +263,13 @@
       </manifest>
       <metainf dir="${hive.root}" includes="LICENSE,NOTICE"/>
     </jar>
+    <ivy:publish settingsRef="${ant.project.name}.ivy.settings"
+                 resolver="local" pubrevision="${version}" overwrite="true"
+                 artifactspattern="${build.dir}/${ivy.publish.pattern}"/>
   </target>
 
   <!-- target to compile tests -->
-  <target name="compile-test" depends="compile">
+  <target name="compile-test" depends="compile,ivy-retrieve-test">
     <echo message="Project: ${ant.project.name}"/>
     <javac
      encoding="${build.encoding}"
@@ -356,17 +371,26 @@
   <target name="test"
   	depends="test-conditions,gen-test,compile-test,test-jar,test-init">
     <echo message="Project: ${ant.project.name}"/>
+    <property name="hadoop.testcp" refid="test.classpath"/>
+    <if>
+      <equals arg1="${hadoop.mr.rev}" arg2="23" />
+      <then>
+        <property name="hadoop.opts" value="${hadoop.opts.23}" />
+      </then>
+      <else>
+        <property name="hadoop.opts" value="${hadoop.opts.20}" />
+      </else>
+    </if>
     <if>
       <equals arg1="${test.print.classpath}" arg2="true" />
       <then>        
-        <property name="testcp" refid="test.classpath"/>
-        <echo message="Test Classpath: ${testcp}"/>
+        <echo message="Test Classpath: ${hadoop.testcp}"/>
       </then>
     </if>
     <junit showoutput="${test.output}" printsummary="yes" haltonfailure="no"
            fork="yes" maxmemory="512m" dir="${basedir}" timeout="${test.timeout}"
            errorProperty="tests.failed" failureProperty="tests.failed" filtertrace="off">
-      <env key="LANG" value="${test.lang}"/> 
+      <env key="HIVE_HADOOP_TEST_CLASSPATH" value="${hadoop.testcp}"/>
       <env key="HADOOP_HOME" value="${hadoop.root}"/>
       <env key="HADOOP_CLASSPATH" value="${test.src.data.dir}/conf:${build.dir.hive}/dist/lib/derby-${derby.version}.jar:${build.dir.hive}/dist/lib/JavaEWAH-${javaewah.version}.jar:${hadoop.root}/modules/*"/> <!-- Modules needed for Hadoop 0.23 -->
       <env key="TZ" value="US/Pacific"/>
@@ -391,8 +415,9 @@
       <sysproperty key="build.ivy.lib.dir" value="${build.ivy.lib.dir}"/>
       <sysproperty key="derby.version" value="${derby.version}"/>
       <sysproperty key="hive.version" value="${version}"/>
+      <sysproperty key="hadoop.bin.path" value="${test.hadoop.bin.path}"/>
 
-      <classpath refid="${test.classpath.id}"/>
+      <classpath refid="test.local.classpath"/>
       <formatter type="${test.junit.output.format}" usefile="${test.junit.output.usefile}" />
       <batchtest todir="${test.build.dir}" unless="testcase">
         <fileset dir="${test.build.classes}"

diff --git a/build.properties b/build.properties
@@ -26,13 +26,17 @@ javac.deprecation=off
 javac.args=
 javac.args.warnings=
 
-hadoop-0.20.version=0.20.1
+hadoop-0.20.version=0.20.2
 hadoop-0.20S.version=1.0.0
-hadoop-0.23.version=0.23.0
+hadoop-0.23.version=0.23.1
 hadoop.version=${hadoop-0.20.version}
 hadoop.security.version=${hadoop-0.20S.version}
 hadoop.mirror=http://mirror.facebook.net/facebook/hive-deps
 hadoop.mirror2=http://archive.cloudera.com/hive-deps
+# Used to determine which set of Hadoop artifacts we depend on.
+# - 20: hadoop-core, hadoop-test
+# - 23: hadoop-common, hadoop-mapreduce-*, etc
+hadoop.mr.rev=20
 
 build.dir.hive=${hive.root}/build
 build.dir.hadoop=${build.dir.hive}/hadoopcore
@@ -41,6 +45,7 @@ build.dir.hadoop=${build.dir.hive}/hadoopcore
 hadoop.version.ant-internal=${hadoop.version}
 hadoop.root.default=${build.dir.hadoop}/hadoop-${hadoop.version.ant-internal}
 hadoop.root=${hadoop.root.default}
+test.hadoop.bin.path=${hive.root}/testutils/hadoop
 # Newer versions of Hadoop name the jar as hadoop-{core,test}-VERSION instead of hadoop-VERSION-{core,test}
 # We will add both styles to the classpath and it will pick up whichever is there
 hadoop.oldstyle-name.jar=${hadoop.root}/hadoop-${hadoop.version.ant-internal}-core.jar
@@ -76,6 +81,9 @@ build.ivy.maven.dir=${build.ivy.dir}/maven
 ivy.conf.dir=${hive.root}/ivy
 ivy.version=2.1.0
 ivy.jar=${build.ivy.lib.dir}/ivy-${ivy.version}.jar
+ivy.changingPattern=.*SNAPSHOT
+ivy.publish.pattern=[artifact]-[revision].[ext]
+ivy.artifact.retrieve.pattern=[conf]/[artifact]-[revision](-[classifier]).[ext]
 ivysettings.xml=${ivy.conf.dir}/ivysettings.xml
 ivyresolvelog=download-only
 ivy.mvn.repo=http://repo2.maven.org/maven2

diff --git a/build.xml b/build.xml
@@ -490,7 +490,14 @@
 
     <!-- copy jar files -->
     <copy todir="${target.lib.dir}" preservelastmodified="true" flatten="true">
-      <fileset dir="${hive.root}" includes="*/*.jar, */*/*.jar" excludes="**/antlr-2*,**/antlr-3*,**/TestSerDe.jar"/>
+      <fileset dir="${hive.root}"  >
+        <include name="*/*.jar"/>
+        <include name="*/*/*.jar"/>
+        <exclude name="**/TestSerDe.jar"/>
+        <exclude name="build/hadoopcore/*.jar"/>
+        <exclude name="**/ant-contrib*.jar"/>
+        <exclude name="**/hive-anttasks*.jar"/>
+      </fileset>
       <fileset file="${build.dir.hive}/cli/hive-cli-${version}.jar"/>
       <fileset file="${build.dir.hive}/common/hive-common-${version}.jar"/>
       <fileset file="${build.dir.hive}/ql/hive-exec-${version}.jar"/>
@@ -501,6 +508,15 @@
       <fileset dir="${build.dir.hive}/ivy/lib/default">
         <include name="*.jar"/>
         <exclude name="*.tar.gz"/>
+        <exclude name="hadoop-*.jar" />
+	<exclude name="**/*high-scale-lib-*"/>
+        <exclude name="**/hamcrest-core-*jar"/>
+        <exclude name="**/junit*.jar"/>
+        <exclude name="**/asm*.jar"/>
+        <exclude name="**/mockito*.jar"/>
+        <exclude name="**/velocity*.jar"/>
+        <exclude name="**/antlr-3*.jar"/>
+        <exclude name="**/antlr-2*.jar"/>
       </fileset>
     </copy>
     <copy todir="${target.example.dir}/files" preservelastmodified="true" flatten="true">
@@ -686,13 +702,6 @@
       <link href="${javadoc.link.java}"/>
 
       <classpath >
-        <fileset dir="${hadoop.root}/lib">
-          <include name="**/*.jar" />
-          <exclude name="**/excluded/" />
-	  <exclude name="**/commons-logging-*.jar" />
-	  <exclude name="**/commons-codec-*.jar" />
-        </fileset>
-	<pathelement path="${hadoop.oldstyle-name.test.jar}"/>
         <path refid="classpath" />
         <pathelement path="${java.class.path}"/>
       </classpath>
@@ -962,7 +971,7 @@
 	     uri="urn:maven-artifact-ant" classpathref="maven-ant-tasks.classpath" />
   </target>
 
-  <target name="maven-build" depends="jar" description="Build Maven artifacts">
+  <target name="maven-build" depends="package" description="Build Maven artifacts">
     <echo message="Project: ${ant.project.name}"/>
     <!-- create jars, poms licences directory -->
     <mkdir dir="${mvn.jar.dir}" />

diff --git a/builtins/build.xml b/builtins/build.xml
@@ -17,30 +17,33 @@
    limitations under the License.
 -->
 
-<project name="builtins" default="jar">
+<project xmlns:ivy="antlib:org.apache.ivy.ant" name="builtins" default="jar">
 
   <property name="src.dir"  location="${basedir}/src"/>
   <import file="../build-common.xml"/>
 
-  <target name="compile" depends="init, setup">
+  <target name="compile" depends="init, setup, ivy-retrieve">
     <echo message="Project: ${ant.project.name}"/>
     <!-- defer compilation until package phase -->
   </target>
 
-  <target name="jar" depends="init">
+  <target name="jar" depends="init, setup, ivy-retrieve">
     <echo message="Project: ${ant.project.name}"/>
     <!-- defer compilation until package phase -->
   </target>
 
-  <target name="package">
+  <target name="package" depends="init, setup, ivy-retrieve">
     <echo message="Project: ${ant.project.name}"/>
     <ant antfile="build-plugin.xml" target="package" inheritAll="false">
       <property name="hive.install.dir" value="${build.dir.hive}/dist"/>
       <property name="hadoop.home" value="${hadoop.root}"/>
     </ant>
+    <ivy:publish settingsRef="${ant.project.name}.ivy.settings"
+                 resolver="local" pubrevision="${version}" overwrite="true"
+                 artifactspattern="${build.dir}/${ivy.publish.pattern}"/>
   </target>
 
-  <target name="test" unless="testcase">
+  <target name="test" unless="testcase" depends="init, setup, ivy-retrieve">
     <echo message="Project: ${ant.project.name}"/>
     <ant antfile="build-plugin.xml" target="test" inheritAll="false">
       <property name="hive.install.dir" value="${build.dir.hive}/dist"/>

diff --git a/builtins/ivy.xml b/builtins/ivy.xml
@@ -23,6 +23,14 @@
         https://cwiki.apache.org/confluence/display/Hive/Home
       </description>
     </info>
-    <dependencies>
-    </dependencies>
+  <configurations>
+    <include file="${ivy.conf.dir}/common-configurations.xml"/>
+  </configurations>
+  <dependencies>
+    <dependency org="org.apache.hive" name="hive-exec" rev="${version}"
+                conf="compile->default" />
+    <dependency org="org.apache.hive" name="hive-pdk" rev="${version}"
+                conf="compile->default" transitive="false" />
+  </dependencies>
+
 </ivy-module>
diff --git a/cli/ivy.xml b/cli/ivy.xml
@@ -28,9 +28,7 @@
   </configurations>
   <dependencies>
     <!-- Runtime Dependencies -->
-    <dependency org="org.apache.hadoop" name="hadoop-core" rev="${hadoop.version.ant-internal}">
-      <artifact name="hadoop" type="source" ext="tar.gz"/>
-    </dependency>
+
     <dependency org="commons-cli" name="commons-cli" rev="${commons-cli.version}"/>
     <dependency org="commons-lang" name="commons-lang" rev="${commons-lang.version}"/>
     <dependency org="commons-logging" name="commons-logging" rev="${commons-logging.version}"
@@ -42,16 +40,8 @@
     <dependency org="org.apache.thrift" name="libthrift" rev="${libthrift.version}"
                 transitive="false"/>
 
-    <dependency org="org.apache.hive" name="hive-common" rev="${version}"
-                conf="runtime" transitive="false"/>
-    <dependency org="org.apache.hive" name="hive-metastore" rev="${version}"
-                conf="runtime" transitive="false"/>
-    <dependency org="org.apache.hive" name="hive-exec" rev="${version}"
-                conf="runtime" transitive="false"/>
-    <dependency org="org.apache.hive" name="hive-serde" rev="${version}"
-                conf="runtime" transitive="false"/>
     <dependency org="org.apache.hive" name="hive-service" rev="${version}"
-                conf="runtime" transitive="false"/>
+                conf="compile->default" />
     <dependency org="org.apache.hive" name="hive-shims" rev="${version}"
                 conf="runtime" transitive="false"/>
     <dependency org="org.apache.hive" name="hive-builtins" rev="${version}"