Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Closes GH-149 create sharelib module with Pig/Streaming JARs

Adding a sharelib module that bundles streaming/pig JARs used by Oozie build.

Updated assemblies to create a sharelib TAR.GZ file

Updated documentation
  • Loading branch information...
commit a81d61da89a8f99b6584e34c6f8ce2da5b3b9a74 1 parent 3d66fca
@tucu00 tucu00 authored
View
11 core/src/main/java/org/apache/oozie/service/WorkflowAppService.java
@@ -229,10 +229,15 @@ public XConfiguration createProtoActionConf(Configuration jobConf, String authTo
*/
private List<String> getLibFiles(FileSystem fs, Path libPath) throws IOException {
List<String> libPaths = new ArrayList<String>();
- FileStatus[] files = fs.listStatus(libPath, new NoPathFilter());
+ if (fs.exists(libPath)) {
+ FileStatus[] files = fs.listStatus(libPath, new NoPathFilter());
- for (FileStatus file : files) {
- libPaths.add((String) file.getPath().toUri().getPath().trim());
+ for (FileStatus file : files) {
+ libPaths.add(file.getPath().toUri().getPath().trim());
+ }
+ }
+ else {
+ XLog.getLog(getClass()).warn("libpath [{0}] does not exists", libPath);
}
return libPaths;
}
View
4 core/src/main/resources/oozie-default.xml
@@ -1157,10 +1157,10 @@
<property>
<name>oozie.service.WorkflowAppService.system.libpath</name>
- <value> </value>
+ <value>/user/${user.name}/share/lib</value>
<description>
System library path to use for workflow applications.
- This path is added to worflow application if their job properties sets
+ This path is added to workflow application if their job properties sets
the property 'oozie.use.system.libpath' to true.
</description>
</property>
View
43 core/src/test/java/org/apache/oozie/service/TestLiteWorkflowAppService.java
@@ -435,4 +435,47 @@ public void testCreateprotoConfWithSystemLibPath() throws Exception {
}
}
+ public void testCreateprotoConfWithMissingSystemLibPath() throws Exception {
+ setSystemProperty(WorkflowAppService.SYSTEM_LIB_PATH, getTestCaseDir() + "/missingsyslib");
+ Services services = new Services();
+ try {
+ services.init();
+ new Services().init();
+
+ Writer writer;
+ Reader reader = IOUtils.getResourceAsReader("wf-schema-valid.xml", -1);
+ writer = new FileWriter(getTestCaseDir() + "/workflow.xml");
+ IOUtils.copyCharStream(reader, writer);
+
+ createTestCaseSubDir("lib");
+ writer = new FileWriter(getTestCaseDir() + "/lib/maputil.jar");
+ writer.write("bla bla");
+ writer.close();
+
+ // using missing system libpath
+ WorkflowAppService wps = Services.get().get(WorkflowAppService.class);
+ Configuration jobConf = new XConfiguration();
+ jobConf.set(OozieClient.APP_PATH, "file://" + getTestCaseDir() + "/workflow.xml");
+ jobConf.set(OozieClient.USER_NAME, getTestUser());
+ jobConf.set(OozieClient.GROUP_NAME, getTestGroup());
+ jobConf.setBoolean(OozieClient.USE_SYSTEM_LIBPATH, true);
+ injectKerberosInfo(jobConf);
+ Configuration protoConf = wps.createProtoActionConf(jobConf, "authToken", true);
+ assertEquals(getTestUser(), protoConf.get(OozieClient.USER_NAME));
+ assertEquals(getTestGroup(), protoConf.get(OozieClient.GROUP_NAME));
+
+ assertEquals(1, protoConf.getStrings(WorkflowAppService.APP_LIB_PATH_LIST).length);
+ List<String> found = new ArrayList<String>();
+ found.add(protoConf.getStrings(WorkflowAppService.APP_LIB_PATH_LIST)[0]);
+ List<String> expected = new ArrayList<String>();
+ expected.add(getTestCaseDir() + "/lib/maputil.jar");
+ Collections.sort(found);
+ Collections.sort(expected);
+ assertEquals(expected, found);
+ }
+ finally {
+ services.destroy();
+ }
+ }
+
}
View
7 docs/src/site/twiki/AG_Install.twiki
@@ -245,6 +245,13 @@ Copy and expand the =oozie-client= TAR.GZ file bundled with the distribution. Ad
Refer to the [[DG_CommandLineTool][Command Line Interface Utilities]] document for a a full reference of the =oozie=
command line tool.
+---++ Oozie Share Lib
+
+The Oozie share lib TAR.GZ file bundled with the distribution contains the necessary files to run Oozie map-reduce
+streaming and pig actions.
+
+The bundled Streaming and Pig JARs are the ones used by Oozie testcases.
+
[[index][::Go back to Oozie Documentation Index::]]
</noautolink>
View
13 docs/src/site/twiki/DG_QuickStart.twiki
@@ -77,6 +77,19 @@ command line tool.
NOTE: The Oozie server installation includes the Oozie client. The Oozie client should be installed in remote machines
only.
+---++ Oozie Share Lib Installation
+
+Expand the =oozie-sharelib= TAR.GZ file bundled with the distribution.
+
+The =share/= directory must be copied to the Oozie HOME directory in HDFS:
+
+<verbatim>
+$ hadoop fs -put share share
+</verbatim>
+
+*NOTE:* This must be done using the Oozie Hadoop (HDFS) user. If an =share= directory already exists in HDFS, it must be
+deleted before copying it again.
+
[[index][::Go back to Oozie Documentation Index::]]
</noautolink>
View
20 docs/src/site/twiki/WorkflowFunctionalSpec.twiki
@@ -16,6 +16,9 @@ Author: Alejandro Abdelnur
---++ Changelog
+---+++!! 2010OCT31
+
+ * #17, Added new section on Shared Libraries
---+++!! 2010APR27
* #3.2.3 Added new "arguments" tag to PIG actions
@@ -2250,6 +2253,23 @@ Any prioritization of jobs in the remote systems is outside of the scope of Oozi
Workflow applications can influence the remote systems priority via configuration if the remote systems support it.
+#ShareLib
+---++ 17 HDFS Share Libraries for Workflow Applications (since Oozie 2.3)
+
+Oozie supports job and system share libraries for workflow jobs.
+
+Share libraries can simplify the deployment and management of common components across workflow applications.
+
+For example, if a workflow job uses a share library with the Streaming, Pig & Har JARs files it does not have to
+bundled those JARs files in the workflow application =lib/= path.
+
+If workflow job uses a share library, Oozie will include all the JAR/SO files in the library in the
+classpath/libpath for all its actions.
+
+A workflow job can specify a share library path using the job property =oozie.libpath=.
+
+A workflow job can use the system share library by setting the job property =oozie.use.system.libpath= to =true=.
+
---++ Appendixes
#OozieWFSchema
View
1  pom.xml
@@ -92,6 +92,7 @@
<module>webapp</module>
<module>examples</module>
<module>docs</module>
+ <module>sharelib</module>
<module>distro</module>
</modules>
View
1  release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 2.3.0 release
+GH-0149 create sharelib module with Pig/Streaming JARs
GH-0119 support for user() EL function in coord apps
GH-0131 add an embedded tomcat in Oozie distribution
GH-0183 update Oozie build to use Hadoop 0.20.104.2
View
103 sharelib/pom.xml
@@ -0,0 +1,103 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Copyright (c) 2010 Yahoo! Inc. All rights reserved.
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+<project>
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>com.yahoo.oozie</groupId> <!-- OOZIE_GROUP_ID -->
+ <artifactId>oozie-main</artifactId>
+ <version>2.3.0-SNAPSHOT</version> <!-- OOZIE_VERSION -->
+ </parent>
+ <artifactId>oozie-sharelib</artifactId>
+ <description>Oozie Share Lib</description>
+ <name>Oozie Share Lib</name>
+ <packaging>jar</packaging>
+
+ <licenses>
+ <license>
+ <name>The Apache Software License, Version 2.0</name>
+ <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
+ </license>
+ </licenses>
+
+ <organization>
+ <name>Yahoo</name>
+ <url>http://www.yahoo.com</url>
+ </organization>
+
+ <dependencies>
+ <dependency>
+ <groupId>${hadoopGroupId}</groupId>
+ <artifactId>hadoop-streaming</artifactId>
+ <version>${hadoopVersion}</version>
+ <scope>compile</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>${hadoopGroupId}</groupId>
+ <artifactId>hadoop-core</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>${pigGroupId}</groupId>
+ <artifactId>${pigArtifactId}</artifactId>
+ <version>${pigVersion}</version>
+ <scope>compile</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>${hadoopGroupId}</groupId>
+ <artifactId>hadoop-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-core</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>jline</groupId>
+ <artifactId>jline</artifactId>
+ <version>0.9.94</version>
+ <scope>compile</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <resources>
+ <resource>
+ <directory>src/main/resources</directory>
+ <filtering>true</filtering>
+ </resource>
+ </resources>
+ <plugins>
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <configuration>
+ <descriptors>
+ <descriptor>../src/main/assemblies/sharelib.xml</descriptor>
+ </descriptors>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+
+</project>
+
View
23 sharelib/src/main/resources/sharelib.properties
@@ -0,0 +1,23 @@
+#
+# Copyright (c) 2010 Yahoo! Inc. All rights reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License. See accompanying LICENSE file.
+
+#
+
+######################
+build.user=${user.name}
+build.time=${build.time}
+build.version=${pom.version}
+vc.revision=${vc.revision}
+vc.url=${vc.url}
+######################
View
7 src/main/assemblies/distro.xml
@@ -119,5 +119,12 @@
<destName>oozie-examples.tar.gz</destName>
<fileMode>0444</fileMode>
</file>
+ <!-- Oozie sharelib TAR.GZ -->
+ <file>
+ <source>${basedir}/../sharelib/target/oozie-sharelib-${project.version}-sharelib.tar.gz</source>
+ <outputDirectory>/</outputDirectory>
+ <destName>oozie-sharelib-${project.version}.tar.gz</destName>
+ <fileMode>0444</fileMode>
+ </file>
</files>
</assembly>
View
37 src/main/assemblies/sharelib.xml
@@ -0,0 +1,37 @@
+<!--
+ Copyright (c) 2010 Yahoo! Inc. All rights reserved.
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+<assembly>
+ <id>sharelib</id>
+ <formats>
+ <format>dir</format>
+ <format>tar.gz</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+
+ <files>
+ <file>
+ <source>${basedir}/target/classes/sharelib.properties</source>
+ <outputDirectory>/share/lib</outputDirectory>
+ </file>
+ </files>
+
+ <dependencySets>
+ <dependencySet>
+ <outputDirectory>/share/lib</outputDirectory>
+ <unpack>false</unpack>
+ </dependencySet>
+ </dependencySets>
+
+</assembly>
Please sign in to comment.
Something went wrong with that request. Please try again.