OOZIE-2481 Add YARN_CONF_DIR in the Shell action (harsh)

core/src/test/java/org/apache/oozie/action/hadoop/TestShellActionExecutor.java

@@ -60,8 +60,11 @@ public class TestShellActionExecutor extends ActionExecutorTestCase {
             : "ls -ltr\necho $1 $2\nexit 1";
     private static final String PERL_SCRIPT_CONTENT = "print \"MY_VAR=TESTING\";";
     private static final String SHELL_SCRIPT_HADOOP_CONF_DIR_CONTENT = Shell.WINDOWS
-            ? "echo OOZIE_ACTION_CONF_XML=%OOZIE_ACTION_CONF_XML%\necho HADOOP_CONF_DIR=%HADOOP_CONF_DIR%"
-            : "echo OOZIE_ACTION_CONF_XML=$OOZIE_ACTION_CONF_XML\necho HADOOP_CONF_DIR=$HADOOP_CONF_DIR";
+            ? "echo OOZIE_ACTION_CONF_XML=%OOZIE_ACTION_CONF_XML%\necho HADOOP_CONF_DIR=%HADOOP_CONF_DIR%\n"
+            : "echo OOZIE_ACTION_CONF_XML=$OOZIE_ACTION_CONF_XML\necho HADOOP_CONF_DIR=$HADOOP_CONF_DIR\n";
+    private static final String SHELL_SCRIPT_YARN_CONF_DIR_CONTENT = Shell.WINDOWS
+            ? "echo OOZIE_ACTION_CONF_XML=%OOZIE_ACTION_CONF_XML%\necho YARN_CONF_DIR=%YARN_CONF_DIR%\n"
+            : "echo OOZIE_ACTION_CONF_XML=$OOZIE_ACTION_CONF_XML\necho YARN_CONF_DIR=$YARN_CONF_DIR\n";
 
     /**
      * Verify if the ShellActionExecutor indeed setups the basic stuffs
@@ -128,6 +131,7 @@ public void testShellScriptHadoopConfDir() throws Exception {
         Path script = new Path(getAppPath(), SHELL_SCRIPTNAME);
         Writer w = new OutputStreamWriter(fs.create(script));
         w.write(SHELL_SCRIPT_HADOOP_CONF_DIR_CONTENT);
+        w.write(SHELL_SCRIPT_YARN_CONF_DIR_CONTENT);
         w.close();
 
         // Create sample Shell action xml
@@ -141,10 +145,12 @@ public void testShellScriptHadoopConfDir() throws Exception {
         WorkflowAction action = _testSubmit(actionXml, true, "");
         String oozieActionConfXml = PropertiesUtils.stringToProperties(action.getData()).getProperty("OOZIE_ACTION_CONF_XML");
         String hadoopConfDir = PropertiesUtils.stringToProperties(action.getData()).getProperty("HADOOP_CONF_DIR");
+        String yarnConfDir = PropertiesUtils.stringToProperties(action.getData()).getProperty("YARN_CONF_DIR");
         assertNotNull(oozieActionConfXml);
         assertNotNull(hadoopConfDir);
         String s = new File(oozieActionConfXml).getParent() + File.separator + "oozie-hadoop-conf-";
         Assert.assertTrue("Expected HADOOP_CONF_DIR to start with " + s + " but was " + hadoopConfDir, hadoopConfDir.startsWith(s));
+        Assert.assertTrue("Expected YARN_CONF_DIR to start with " + s + " but was " + yarnConfDir, yarnConfDir.startsWith(s));
     }
 
     /**

docs/src/site/twiki/DG_ShellActionExtension.twiki

@@ -219,6 +219,10 @@ nameNode=hdfs://localhost:8020
 queueName=default
 </verbatim>
 
+---+++ Shell Action Configuration
+
+=oozie.action.shell.setup.hadoop.conf.dir= - Generates a config directory with various core/hdfs/yarn/mapred-site.xml files and points =HADOOP_CONF_DIR= and =YARN_CONF_DIR= env-vars to it, before the Script is invoked. XML is sourced from the action configuration. Useful when the Shell script passed uses various =hadoop= commands. Default is false.
+
 ---+++ Shell Action Logging
 
 Shell action's stdout and stderr output are redirected to the Oozie Launcher map-reduce job task STDOUT that runs the shell command.

release-log.txt

@@ -1,5 +1,6 @@
 -- Oozie 4.3.0 release (trunk - unreleased)
 
+OOZIE-2481 Add YARN_CONF_DIR in the Shell action (harsh)
 OOZIE-2492 JSON security issue in js code (fdenes via rkanter)
 OOZIE-2429 TestEventGeneration test is flakey (fdenes via rkanter)
 OOZIE-2466 Repeated failure of TestMetricsInstrumentation.testSamplers (fdenes via rkanter)

sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/ShellMain.java

@@ -41,6 +41,7 @@ public class ShellMain extends LauncherMain {
     public static final String CONF_OOZIE_SHELL_SETUP_HADOOP_CONF_DIR = "oozie.action.shell.setup.hadoop.conf.dir";
     public static final String OOZIE_ACTION_CONF_XML = "OOZIE_ACTION_CONF_XML";
     private static final String HADOOP_CONF_DIR = "HADOOP_CONF_DIR";
+    private static final String YARN_CONF_DIR = "YARN_CONF_DIR";
 
     private static String[] HADOOP_SITE_FILES = new String[] {"core-site.xml", "hdfs-site.xml", "mapred-site.xml", "yarn-site.xml"};
 
@@ -117,9 +118,9 @@ private int execute(Configuration actionConf) throws Exception {
 
     /**
      * This method takes the OOZIE_ACTION_CONF_XML and copies it to Hadoop *-site files in a new directory; it then sets the
-     * HADOOP_CONF_DIR to point there.  This should allow most Hadoop ecosystem CLI programs to have the proper configuration,
+     * HADOOP/YARN_CONF_DIR to point there.  This should allow most Hadoop ecosystem CLI programs to have the proper configuration,
      * propagated from Oozie's copy and including anything set in the Workflow's configuration section as well.  Otherwise,
-     * HADOOP_CONF_DIR points to the NodeManager's *-site files, which are likely not suitable for client programs.
+     * HADOOP/YARN_CONF_DIR points to the NodeManager's *-site files, which are likely not suitable for client programs.
      * It will only do this if {@link CONF_OOZIE_SHELL_SETUP_HADOOP_CONF_DIR} is set to true.
      *
      * @param actionConf The action configuration
@@ -140,8 +141,10 @@ private void prepareHadoopConfigs(Configuration actionConf, Map<String, String>
                     dstFiles[i] = new File(confDir, HADOOP_SITE_FILES[i]);
                 }
                 copyFileMultiplex(actionXmlFile, dstFiles);
-                System.out.println("Setting " + HADOOP_CONF_DIR + " to " + confDir.getAbsolutePath());
+                System.out.println("Setting " + HADOOP_CONF_DIR + " and " + YARN_CONF_DIR
+                    + " to " + confDir.getAbsolutePath());
                 envp.put(HADOOP_CONF_DIR, confDir.getAbsolutePath());
+                envp.put(YARN_CONF_DIR, confDir.getAbsolutePath());
             }
         }
     }