Skip to content

Commit

Permalink
HIVE-2244. Add a Plugin Developer Kit to Hive (John Sichi via cws)
Browse files Browse the repository at this point in the history
git-svn-id: https://svn.apache.org/repos/asf/hive/trunk@1177382 13f79535-47bb-0310-9956-ffa450edef68
  • Loading branch information
cwsteinbach committed Sep 29, 2011
1 parent fe9f7b1 commit c1c13d0
Show file tree
Hide file tree
Showing 15 changed files with 693 additions and 10 deletions.
20 changes: 17 additions & 3 deletions build.xml
Expand Up @@ -119,7 +119,7 @@
<subant target="@{target}">
<property name="build.dir.hive" location="${build.dir.hive}"/>
<property name="is-offline" value="${is-offline}"/>
<filelist dir="." files="ant/build.xml,shims/build.xml,common/build.xml,serde/build.xml,metastore/build.xml,ql/build.xml,contrib/build.xml,service/build.xml,cli/build.xml,jdbc/build.xml,hwi/build.xml,hbase-handler/build.xml,ant/build.xml"/>
<filelist dir="." files="ant/build.xml,shims/build.xml,common/build.xml,serde/build.xml,metastore/build.xml,ql/build.xml,contrib/build.xml,service/build.xml,cli/build.xml,jdbc/build.xml,hwi/build.xml,hbase-handler/build.xml,ant/build.xml,pdk/build.xml"/>
</subant>
</sequential>
</macrodef>
Expand All @@ -130,7 +130,7 @@
<subant target="@{target}">
<property name="build.dir.hive" location="${build.dir.hive}"/>
<property name="is-offline" value="${is-offline}"/>
<filelist dir="." files="shims/build.xml,common/build.xml,serde/build.xml,metastore/build.xml,ql/build.xml,contrib/build.xml,service/build.xml,cli/build.xml,jdbc/build.xml,hwi/build.xml,hbase-handler/build.xml"/>
<filelist dir="." files="shims/build.xml,common/build.xml,serde/build.xml,metastore/build.xml,ql/build.xml,contrib/build.xml,service/build.xml,cli/build.xml,jdbc/build.xml,hwi/build.xml,hbase-handler/build.xml,pdk/build.xml"/>
</subant>
</sequential>
</macrodef>
Expand Down Expand Up @@ -292,6 +292,7 @@
<mkdir dir="${target.conf.dir}"/>
<mkdir dir="${target.bin.dir}"/>
<mkdir dir="${target.scripts.dir}/metastore/upgrade"/>
<mkdir dir="${target.scripts.dir}/pdk"/>
<mkdir dir="${target.example.dir}"/>
<mkdir dir="${target.example.dir}/files"/>
<mkdir dir="${target.example.dir}/queries"/>
Expand All @@ -301,6 +302,9 @@
<copy todir="${target.scripts.dir}/metastore/upgrade">
<fileset dir="${hive.root}/metastore/scripts/upgrade" excludes="${vcs.excludes}"/>
</copy>
<copy todir="${target.scripts.dir}/pdk">
<fileset dir="${hive.root}/pdk/scripts" excludes="${vcs.excludes},README"/>
</copy>
<copy todir="${target.bin.dir}/ext">
<fileset dir="${hive.root}/bin/ext" excludes="${vcs.excludes}"/>
</copy>
Expand Down Expand Up @@ -373,6 +377,9 @@
<copy todir="${target.example.dir}/files" preservelastmodified="true" flatten="true">
<fileset dir="${test.data.dir}/files" includes="*.*" excludes="${vcs.excludes}"/>
</copy>
<copy todir="${target.example.dir}/test-plugin">
<fileset dir="${hive.root}/pdk/test-plugin"/>
</copy>
<copy file="${basedir}/README.txt" todir="${target.dir}">
<filterset>
<filter token="VERSION" value="${version}"/>
Expand Down Expand Up @@ -452,7 +459,6 @@
</fileset>
<mapper type="glob" from="*.launchtemplate" to="*.launch"/>
</move>

</target>

<target name="clean-eclipse-files"
Expand Down Expand Up @@ -507,6 +513,7 @@
<packageset dir="ql/src/gen/thrift/gen-javabean"/>
<packageset dir="${build.dir.hive}/ql/gen/antlr/gen-java"/>
<packageset dir="shims/src/common/java"/>
<packageset dir="pdk/src/java"/>

<link href="${javadoc.link.java}"/>

Expand Down Expand Up @@ -691,6 +698,8 @@
todir="${build.dir.hive}/maven/jars/" />
<copy file="${build.dir.hive}/shims/hive-shims-${version}.jar"
todir="${build.dir.hive}/maven/jars/" />
<copy file="${build.dir.hive}/pdk/hive-pdk-${version}.jar"
todir="${build.dir.hive}/maven/jars/" />

<!-- copy over maven pom files created using the make-pom target and rename to maven convention -->
<copy file="${build.dir.hive}/anttasks/pom.xml"
Expand All @@ -717,6 +726,8 @@
tofile="${build.dir.hive}/maven/poms/hive-service-${version}.pom" />
<copy file="${build.dir.hive}/shims/pom.xml"
tofile="${build.dir.hive}/maven/poms/hive-shims-${version}.pom" />
<copy file="${build.dir.hive}/pdk/pom.xml"
tofile="${build.dir.hive}/maven/poms/hive-pdk-${version}.pom" />

<!-- copy over licence -->
<copy file="${hive.root}/LICENSE" todir="${build.dir.hive}/maven/licences/" />
Expand Down Expand Up @@ -776,6 +787,9 @@
<antcall target="maven-publish-artifact">
<param name="hive.project" value="hwi" />
</antcall>
<antcall target="maven-publish-artifact">
<param name="hive.project" value="pdk" />
</antcall>
<antcall target="maven-publish-artifact">
<param name="hive.project" value="jdbc" />
</antcall>
Expand Down
19 changes: 12 additions & 7 deletions cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java
Expand Up @@ -549,10 +549,15 @@ public int complete (String buffer, int offset, List completions) {
}

public static void main(String[] args) throws Exception {
int ret = run(args);
System.exit(ret);
}

public static int run(String[] args) throws Exception {

OptionsProcessor oproc = new OptionsProcessor();
if (!oproc.process_stage1(args)) {
System.exit(1);
return 1;
}

// NOTE: It is critical to do this here so that log4j is reinitialized
Expand All @@ -572,11 +577,11 @@ public static void main(String[] args) throws Exception {
ss.out = new PrintStream(System.out, true, "UTF-8");
ss.err = new PrintStream(System.err, true, "UTF-8");
} catch (UnsupportedEncodingException e) {
System.exit(3);
return 3;
}

if (!oproc.process_stage2(ss)) {
System.exit(2);
return 2;
}

if (!ss.getIsSilent()) {
Expand Down Expand Up @@ -627,16 +632,16 @@ public static void main(String[] args) throws Exception {
cli.processInitFiles(ss);

if (ss.execString != null) {
System.exit(cli.processLine(ss.execString));
return cli.processLine(ss.execString);
}

try {
if (ss.fileName != null) {
System.exit(cli.processFile(ss.fileName));
return cli.processFile(ss.fileName);
}
} catch (FileNotFoundException e) {
System.err.println("Could not open input file for reading. (" + e.getMessage() + ")");
System.exit(3);
return 3;
}

ConsoleReader reader = new ConsoleReader();
Expand Down Expand Up @@ -675,7 +680,7 @@ public static void main(String[] args) throws Exception {

ss.close();

System.exit(ret);
return ret;
}

/**
Expand Down
59 changes: 59 additions & 0 deletions pdk/build.xml
@@ -0,0 +1,59 @@
<?xml version="1.0" encoding="UTF-8"?>

<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->

<project name="pdk" default="jar">

<property name="src.dir" location="${basedir}/src/java"/>
<import file="../build-common.xml"/>

<path id="classpath-pdk">
<fileset dir="${hive.root}/testlibs" includes="junit-3.8.1.jar"/>
<path refid="classpath"/>
</path>

<target name="compile" depends="init, setup">
<echo message="Compiling: ${ant.project.name}"/>
<javac
encoding="${build.encoding}"
srcdir="${src.dir}"
includes="**/*.java"
destdir="${build.classes}"
debug="${javac.debug}"
deprecation="${javac.deprecation}">
<compilerarg line="${javac.args} ${javac.args.warnings}" />
<classpath refid="classpath-pdk"/>
</javac>
</target>

<target name="test" unless="testcase">
<delete quiet="true" dir="${build.dir}/test-plugin"/>
<copy todir="${build.dir}/test-plugin">
<fileset dir="${basedir}/test-plugin"/>
</copy>
<ant antfile="${build.dir}/test-plugin/build.xml"
target="package" inheritAll="false" inheritRefs="false">
<property name="hive.install.dir" value="${build.dir.hive}/dist"/>
</ant>
<ant antfile="${build.dir}/test-plugin/build.xml"
target="test" inheritAll="false" inheritRefs="false">
<property name="hive.install.dir" value="${build.dir.hive}/dist"/>
</ant>
</target>

</project>
3 changes: 3 additions & 0 deletions pdk/scripts/README
@@ -0,0 +1,3 @@
Note that this directory contains scripts which are bundled into the
Plugin Development Kit (rather than used as part of the Hive build
itself).
133 changes: 133 additions & 0 deletions pdk/scripts/build-plugin.xml
@@ -0,0 +1,133 @@
<?xml version="1.0"?>

<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->

<project name="hive-plugin-import" default="intended-for-import-only">
<dirname property="pdk.script.dir" file="${ant.file.hive-plugin-import}"/>
<property name="plugin.dir" location="${basedir}"/>
<property name="src.dir" location="${plugin.dir}/src"/>
<property name="build.dir" location="${plugin.dir}/build"/>
<property name="build.classes" location="${build.dir}/classes"/>
<property name="build.metadata" location="${build.dir}/metadata"/>
<property name="install.dir" location="${pdk.script.dir}/../.."/>
<property name="function.sql.prefix" value=""/>
<property name="plugin.jar.basename"
value="${plugin.libname}-${plugin.version}.jar"/>

<property environment="env"/>

<path id="plugin.classpath">
<fileset dir="${install.dir}" includes="lib/hive-exec-*.jar"/>
<fileset dir="${install.dir}" includes="lib/hive-pdk-*.jar"/>
<fileset dir="${install.dir}" includes="lib/commons-lang-*.jar"/>
<fileset dir="${install.dir}" includes="lib/commons-logging-*.jar"/>
<fileset dir="${env.HADOOP_HOME}" includes="hadoop-*-core.jar"/>
<pathelement location="${build.classes}"/>
</path>

<path id="plugin.test.classpath">
<pathelement path="${install.dir}/conf"/>
<fileset dir="${install.dir}" includes="lib/*.jar"/>
<fileset dir="${env.HADOOP_HOME}" includes="hadoop-*-core.jar"/>
<fileset dir="${build.dir}" includes="${plugin.jar.basename}"/>
</path>

<target name="compile">
<mkdir dir="${build.classes}"/>
<javac
srcdir="${src.dir}"
includes="**/*.java"
includeantruntime="false"
destdir="${build.classes}">
<classpath refid="plugin.classpath"/>
</javac>
</target>

<target name="jar" depends="compile">
<jar
jarfile="${build.dir}/${plugin.jar.basename}"
basedir="${build.classes}">
<manifest>
<attribute name="Implementation-Title" value="${plugin.title}"/>
<attribute name="Implementation-Version" value="${plugin.version}"/>
<attribute name="Implementation-Vendor" value="${plugin.vendor}"/>
</manifest>
</jar>
</target>

<target name="get-class-list">
<path id="class.file.list">
<fileset dir="${build.classes}">
<include name="**/*.class"/>
</fileset>
</path>
<pathconvert property="class.list"
pathsep=" "
dirsep="."
refid="class.file.list">
<map from="${build.classes}/" to=""/>
<mapper>
<chainedmapper>
<globmapper from="*.class" to="*"/>
</chainedmapper>
</mapper>
</pathconvert>
</target>

<target name="extract-functions" depends="compile, get-class-list">
<mkdir dir="${build.metadata}"/>
<java classname="org.apache.hive.pdk.FunctionExtractor"
classpathref="plugin.classpath" fork="true"
output="${build.metadata}/class-info.xml">
<arg line="${class.list}" />
</java>
<xslt style="${pdk.script.dir}/class-registration.xsl"
force="true"
in="${build.metadata}/class-info.xml"
out="${build.metadata}/class-registration.sql">
<param name="functionPrefix" expression="${function.sql.prefix}"/>
</xslt>
<echo file="${build.metadata}/add-jar.sql"
message="ADD JAR ${plugin.jar.basename};" />
</target>

<target name="clean">
<delete quiet="true" dir="${build.dir}"/>
</target>

<target name="package" depends="jar,extract-functions"/>

<target name="test" depends="package" unless="testcase">
<junit fork="true" printsummary="on" haltonfailure="true"
dir="${build.dir}">
<classpath refid="plugin.test.classpath"/>
<sysproperty key="hive.plugin.root.dir" value="${plugin.dir}"/>
<sysproperty key="hive.plugin.class.list" value="${class.list}"/>
<env key="HADOOP_HOME" value="${env.HADOOP_HOME}"/>
<env key="HIVE_CONF_DIR" value="${install.dir}/conf"/>
<env key="HIVE_HOME" value="${install.dir}"/>
<env key="HIVE_PLUGIN_ROOT_DIR" value="${plugin.dir}"/>
<formatter type="plain"/>
<test name="org.apache.hive.pdk.PluginTest" todir="${build.dir}/"/>
<assertions>
<enable />
</assertions>
</junit>
</target>

</project>
41 changes: 41 additions & 0 deletions pdk/scripts/class-registration.xsl
@@ -0,0 +1,41 @@
<?xml version="1.0"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<xsl:stylesheet
version="2.0"
xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
>
<xsl:output method="text" indent="no" />

<xsl:param name="functionPrefix"/>

<xsl:template match="Class">
<xsl:text>CREATE TEMPORARY FUNCTION </xsl:text>
<xsl:value-of select="$functionPrefix"/>
<xsl:value-of select="@sqlname"/>
<xsl:text> AS '</xsl:text>
<xsl:value-of select="@javaname"/>
<xsl:text>';</xsl:text>
<xsl:text>
</xsl:text>
</xsl:template>

<xsl:template match="/ | @* | node()">
<xsl:apply-templates select="@* | node()" />
</xsl:template>

</xsl:stylesheet>

0 comments on commit c1c13d0

Please sign in to comment.