Closes GH-149 create sharelib module with Pig/Streaming JARs

Adding a sharelib module that bundles streaming/pig JARs used by Oozie build. Updated assemblies to create a sharelib TAR.GZ file Updated documentation
YahooArchive · Nov 6, 2010 · a81d61d · a81d61d
1 parent 3d66fca
commit a81d61d
Show file tree

Hide file tree

Showing 12 changed files with 265 additions and 5 deletions.
diff --git a/core/src/main/java/org/apache/oozie/service/WorkflowAppService.java b/core/src/main/java/org/apache/oozie/service/WorkflowAppService.java
@@ -229,10 +229,15 @@ public XConfiguration createProtoActionConf(Configuration jobConf, String authTo
      */
     private List<String> getLibFiles(FileSystem fs, Path libPath) throws IOException {
         List<String> libPaths = new ArrayList<String>();
-        FileStatus[] files = fs.listStatus(libPath, new NoPathFilter());
+        if (fs.exists(libPath)) {
+            FileStatus[] files = fs.listStatus(libPath, new NoPathFilter());
 
-        for (FileStatus file : files) {
-            libPaths.add((String) file.getPath().toUri().getPath().trim());
+            for (FileStatus file : files) {
+                libPaths.add(file.getPath().toUri().getPath().trim());
+            }
+        }
+        else {
+            XLog.getLog(getClass()).warn("libpath [{0}] does not exists", libPath);
         }
         return libPaths;
     }

diff --git a/core/src/main/resources/oozie-default.xml b/core/src/main/resources/oozie-default.xml
@@ -1157,10 +1157,10 @@
 
     <property>
         <name>oozie.service.WorkflowAppService.system.libpath</name>
-        <value> </value>
+        <value>/user/${user.name}/share/lib</value>
         <description>
             System library path to use for workflow applications.
-            This path is added to worflow application if their job properties sets
+            This path is added to workflow application if their job properties sets
             the property 'oozie.use.system.libpath' to true.
         </description>
     </property>

diff --git a/core/src/test/java/org/apache/oozie/service/TestLiteWorkflowAppService.java b/core/src/test/java/org/apache/oozie/service/TestLiteWorkflowAppService.java
@@ -435,4 +435,47 @@ public void testCreateprotoConfWithSystemLibPath() throws Exception {
         }
     }
 
+    public void testCreateprotoConfWithMissingSystemLibPath() throws Exception {
+        setSystemProperty(WorkflowAppService.SYSTEM_LIB_PATH, getTestCaseDir() + "/missingsyslib");
+        Services services = new Services();
+        try {
+            services.init();
+            new Services().init();
+
+            Writer writer;
+            Reader reader = IOUtils.getResourceAsReader("wf-schema-valid.xml", -1);
+            writer = new FileWriter(getTestCaseDir() + "/workflow.xml");
+            IOUtils.copyCharStream(reader, writer);
+
+            createTestCaseSubDir("lib");
+            writer = new FileWriter(getTestCaseDir() + "/lib/maputil.jar");
+            writer.write("bla bla");
+            writer.close();
+
+            // using missing system libpath
+            WorkflowAppService wps = Services.get().get(WorkflowAppService.class);
+            Configuration jobConf = new XConfiguration();
+            jobConf.set(OozieClient.APP_PATH, "file://" + getTestCaseDir() + "/workflow.xml");
+            jobConf.set(OozieClient.USER_NAME, getTestUser());
+            jobConf.set(OozieClient.GROUP_NAME, getTestGroup());
+            jobConf.setBoolean(OozieClient.USE_SYSTEM_LIBPATH, true);
+            injectKerberosInfo(jobConf);
+            Configuration protoConf = wps.createProtoActionConf(jobConf, "authToken", true);
+            assertEquals(getTestUser(), protoConf.get(OozieClient.USER_NAME));
+            assertEquals(getTestGroup(), protoConf.get(OozieClient.GROUP_NAME));
+
+            assertEquals(1, protoConf.getStrings(WorkflowAppService.APP_LIB_PATH_LIST).length);
+            List<String> found = new ArrayList<String>();
+            found.add(protoConf.getStrings(WorkflowAppService.APP_LIB_PATH_LIST)[0]);
+            List<String> expected = new ArrayList<String>();
+            expected.add(getTestCaseDir() + "/lib/maputil.jar");
+            Collections.sort(found);
+            Collections.sort(expected);
+            assertEquals(expected, found);
+        }
+        finally {
+            services.destroy();
+        }
+    }
+
 }
diff --git a/docs/src/site/twiki/AG_Install.twiki b/docs/src/site/twiki/AG_Install.twiki
@@ -245,6 +245,13 @@ Copy and expand the =oozie-client= TAR.GZ file bundled with the distribution. Ad
 Refer to the [[DG_CommandLineTool][Command Line Interface Utilities]] document for a a full reference of the =oozie=
 command line tool.
 
+---++ Oozie Share Lib
+
+The Oozie share lib TAR.GZ file bundled with the distribution contains the necessary files to run Oozie map-reduce
+streaming and pig actions.
+
+The bundled Streaming and Pig JARs are the ones used by Oozie testcases.
+
 [[index][::Go back to Oozie Documentation Index::]]
 
 </noautolink>
diff --git a/docs/src/site/twiki/DG_QuickStart.twiki b/docs/src/site/twiki/DG_QuickStart.twiki
@@ -77,6 +77,19 @@ command line tool.
 NOTE: The Oozie server installation includes the Oozie client. The Oozie client should be installed in remote machines
 only.
 
+---++ Oozie Share Lib Installation
+
+Expand the =oozie-sharelib= TAR.GZ file bundled with the distribution.
+
+The =share/= directory must be copied to the Oozie HOME directory in HDFS:
+
+<verbatim>
+$ hadoop fs -put share share
+</verbatim>
+
+*NOTE:* This must be done using the Oozie Hadoop (HDFS) user. If an =share= directory already exists in HDFS, it must be
+deleted before copying it again.
+
 [[index][::Go back to Oozie Documentation Index::]]
 
 </noautolink>
diff --git a/docs/src/site/twiki/WorkflowFunctionalSpec.twiki b/docs/src/site/twiki/WorkflowFunctionalSpec.twiki
@@ -16,6 +16,9 @@ Author: Alejandro Abdelnur
 
 ---++ Changelog
 
+---+++!! 2010OCT31
+
+   * #17, Added new section on Shared Libraries
 ---+++!! 2010APR27
 
    * #3.2.3 Added new "arguments" tag to PIG actions
@@ -2250,6 +2253,23 @@ Any prioritization of jobs in the remote systems is outside of the scope of Oozi
 
 Workflow applications can influence the remote systems priority via configuration if the remote systems support it.
 
+#ShareLib
+---++ 17 HDFS Share Libraries for Workflow Applications (since Oozie 2.3)
+
+Oozie supports job and system share libraries for workflow jobs.
+
+Share libraries can simplify the deployment and management of common components across workflow applications.
+
+For example, if a workflow job uses a share library with the Streaming, Pig & Har JARs files it does not have to
+bundled those JARs files in the workflow application =lib/= path.
+
+If workflow job uses a share library, Oozie will include all the JAR/SO files in the library in the
+classpath/libpath for all its actions.
+
+A workflow job can specify a share library path using the job property =oozie.libpath=.
+
+A workflow job can use the system share library by setting the job property =oozie.use.system.libpath= to =true=.
+
 ---++ Appendixes
 
 #OozieWFSchema

diff --git a/pom.xml b/pom.xml
@@ -92,6 +92,7 @@
         <module>webapp</module>
         <module>examples</module>
         <module>docs</module>
+        <module>sharelib</module>
         <module>distro</module>
     </modules>
 

diff --git a/release-log.txt b/release-log.txt
@@ -1,5 +1,6 @@
 -- Oozie 2.3.0 release
 
+GH-0149 create sharelib module with Pig/Streaming JARs
 GH-0119 support for user() EL function in coord apps
 GH-0131 add an embedded tomcat in Oozie distribution
 GH-0183 update Oozie build to use Hadoop 0.20.104.2

diff --git a/sharelib/pom.xml b/sharelib/pom.xml
@@ -0,0 +1,103 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Copyright (c) 2010 Yahoo! Inc. All rights reserved.
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+<project>
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>com.yahoo.oozie</groupId> <!-- OOZIE_GROUP_ID -->
+        <artifactId>oozie-main</artifactId>
+        <version>2.3.0-SNAPSHOT</version> <!-- OOZIE_VERSION -->
+    </parent>
+    <artifactId>oozie-sharelib</artifactId>
+    <description>Oozie Share Lib</description>
+    <name>Oozie Share Lib</name>
+    <packaging>jar</packaging>
+
+    <licenses>
+        <license>
+            <name>The Apache Software License, Version 2.0</name>
+            <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
+        </license>
+    </licenses>
+
+    <organization>
+        <name>Yahoo</name>
+        <url>http://www.yahoo.com</url>
+    </organization>
+
+    <dependencies>
+        <dependency>
+            <groupId>${hadoopGroupId}</groupId>
+            <artifactId>hadoop-streaming</artifactId>
+            <version>${hadoopVersion}</version>
+            <scope>compile</scope>
+            <exclusions>
+                <exclusion>
+                    <groupId>${hadoopGroupId}</groupId>
+                    <artifactId>hadoop-core</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>${pigGroupId}</groupId>
+            <artifactId>${pigArtifactId}</artifactId>
+            <version>${pigVersion}</version>
+            <scope>compile</scope>
+            <exclusions>
+                <exclusion>
+                    <groupId>${hadoopGroupId}</groupId>
+                    <artifactId>hadoop-core</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.hadoop</groupId>
+                    <artifactId>hadoop-core</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>jline</groupId>
+            <artifactId>jline</artifactId>
+            <version>0.9.94</version>
+            <scope>compile</scope>
+            <exclusions>
+                <exclusion>
+                    <groupId>junit</groupId>
+                    <artifactId>junit</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <resources>
+            <resource>
+                <directory>src/main/resources</directory>
+                <filtering>true</filtering>
+            </resource>
+        </resources>
+        <plugins>
+            <plugin>
+                <artifactId>maven-assembly-plugin</artifactId>
+                <configuration>
+                    <descriptors>
+                        <descriptor>../src/main/assemblies/sharelib.xml</descriptor>
+                    </descriptors>
+                </configuration>
+            </plugin>
+        </plugins>
+    </build>
+
+</project>
+
diff --git a/sharelib/src/main/resources/sharelib.properties b/sharelib/src/main/resources/sharelib.properties
@@ -0,0 +1,23 @@
+#
+# Copyright (c) 2010 Yahoo! Inc. All rights reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License. See accompanying LICENSE file.
+
+#
+
+######################
+build.user=${user.name}
+build.time=${build.time}
+build.version=${pom.version}
+vc.revision=${vc.revision}
+vc.url=${vc.url}
+######################
diff --git a/src/main/assemblies/distro.xml b/src/main/assemblies/distro.xml
@@ -119,5 +119,12 @@
             <destName>oozie-examples.tar.gz</destName>
             <fileMode>0444</fileMode>
         </file>
+        <!-- Oozie sharelib TAR.GZ  -->
+        <file>
+            <source>${basedir}/../sharelib/target/oozie-sharelib-${project.version}-sharelib.tar.gz</source>
+            <outputDirectory>/</outputDirectory>
+            <destName>oozie-sharelib-${project.version}.tar.gz</destName>
+            <fileMode>0444</fileMode>
+        </file>
     </files>
 </assembly>
diff --git a/src/main/assemblies/sharelib.xml b/src/main/assemblies/sharelib.xml
@@ -0,0 +1,37 @@
+<!--
+  Copyright (c) 2010 Yahoo! Inc. All rights reserved.
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+<assembly>
+    <id>sharelib</id>
+    <formats>
+        <format>dir</format>
+        <format>tar.gz</format>
+    </formats>
+    <includeBaseDirectory>false</includeBaseDirectory>
+
+    <files>
+        <file>
+            <source>${basedir}/target/classes/sharelib.properties</source>
+            <outputDirectory>/share/lib</outputDirectory>
+        </file>
+    </files>
+
+    <dependencySets>
+        <dependencySet>
+            <outputDirectory>/share/lib</outputDirectory>
+            <unpack>false</unpack>
+        </dependencySet>
+    </dependencySets>
+
+</assembly>