FALCON-2047 Use /apps/falcon/extensions/mirroring as base dir for Hiv…

…eDR instead of /apps/data-mirroring/ Author: Sowmya Ramesh <sramesh@hortonworks.com> Reviewers: "Venkat Ranganathan <venkat@hortonworks.com>, Balu Vellanki <balu@apache.org>, Peeyush B <pbishnoi@hortonworks.com>, Praveen Adlakha <adlakha.praveen@gmail.com>" Closes apache#252 from sowmyaramesh/FALCON-2047
pallavi-rao · Feb 16, 2018 · 787587f · 787587f
1 parent 0a184a9
commit 787587f
Show file tree

Hide file tree

Showing 13 changed files with 41 additions and 22 deletions.
diff --git a/addons/hivedr/src/main/java/org/apache/falcon/hive/HiveDRTool.java b/addons/hivedr/src/main/java/org/apache/falcon/hive/HiveDRTool.java
@@ -136,13 +136,6 @@ private void init(String[] args) throws Exception {
         inputOptions = parseOptions(args);
         LOG.info("Input Options: {}", inputOptions);
 
-        // Update the source staging path
-        inputOptions.setSourceStagingPath();
-        inputOptions.setTargetStagingPath();
-
-        LOG.info("srcStaginPath: {}", inputOptions.getSourceStagingPath());
-        LOG.info("tgtStaginPath: {}", inputOptions.getTargetStagingPath());
-
         Configuration sourceConf = FileUtils.getConfiguration(getConf(), inputOptions.getSourceWriteEP(),
                 inputOptions.getSourceNNKerberosPrincipal());
         sourceClusterFS = FileSystem.get(sourceConf);
@@ -155,6 +148,14 @@ private void init(String[] args) throws Exception {
 
         // init DR status store
         drStore = new HiveDRStatusStore(targetClusterFS);
+
+        // Update the source staging path after initing DR status store
+        inputOptions.setSourceStagingPath();
+        inputOptions.setTargetStagingPath();
+
+        LOG.info("srcStaginPath: {}", inputOptions.getSourceStagingPath());
+        LOG.info("tgtStaginPath: {}", inputOptions.getTargetStagingPath());
+
         eventSoucerUtil = new EventSourcerUtils(jobConf, inputOptions.shouldKeepHistory(), inputOptions.getJobName());
     }
 
@@ -310,7 +311,7 @@ private String getLastEvents(Configuration conf) throws Exception {
     }
 
     private Map<String, Long> getLastDBTableEvents(Path lastEventIdFile) throws Exception {
-        Map<String, Long> lastEventsIdMap = new HashMap<String, Long>();
+        Map<String, Long> lastEventsIdMap = new HashMap<>();
         BufferedReader in = new BufferedReader(new InputStreamReader(jobFS.open(lastEventIdFile)));
         try {
             String line;

diff --git a/addons/hivedr/src/main/java/org/apache/falcon/hive/util/DRStatusStore.java b/addons/hivedr/src/main/java/org/apache/falcon/hive/util/DRStatusStore.java
@@ -30,7 +30,7 @@
  */
 public abstract class DRStatusStore {
 
-    public static final String BASE_DEFAULT_STORE_PATH = "/apps/data-mirroring/";
+    public static final String BASE_DEFAULT_STORE_PATH = "/apps/falcon/extensions/mirroring";
     public static final FsPermission DEFAULT_STORE_PERMISSION =
             new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.NONE);
 

diff --git a/addons/hivedr/src/main/java/org/apache/falcon/hive/util/EventUtils.java b/addons/hivedr/src/main/java/org/apache/falcon/hive/util/EventUtils.java
@@ -50,7 +50,7 @@
 import java.util.Properties;
 
 /**
- * Utility class to handle Hive events for data-mirroring.
+ * Utility class to handle Hive events for hive-mirroring.
  */
 public class EventUtils {
     private static final String DRIVER_NAME = "org.apache.hive.jdbc.HiveDriver";

diff --git a/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java b/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java
@@ -151,16 +151,14 @@ public String getExtensionLibPath(final String extensionName) throws StoreAccess
 
             for (FileStatus fileStatus : files) {
                 if (fileStatus.getPath().getName().equalsIgnoreCase(LIBS_DIR)) {
-                    if (fileStatus.getLen() != 0) {
                         libsPath = Path.getPathWithoutSchemeAndAuthority(fileStatus.getPath());
-                    }
                     break;
                 }
             }
 
             if (libsPath == null) {
                 LOG.info("For extension " + extensionName + " there is no "
-                        + LIBS_DIR + "at the extension store path " + storePath);
+                        + LIBS_DIR + "at the extension store path " + extensionPath);
                 return null;
             } else {
                 return libsPath.toString();

diff --git a/extensions/src/test/resources/hive-mirroring-template.xml b/extensions/src/test/resources/hive-mirroring-template.xml
@@ -37,8 +37,8 @@
         <property name="oozie.wf.subworkflow.classpath.inheritance" value="true"/>
     </properties>
 
-    <workflow name="##workflow.name##" engine="oozie"
-              path="/apps/data-mirroring/workflows/hive-disaster-recovery-workflow.xml" lib="##workflow.lib.path##"/>
+    <workflow name="##workflow.name##" engine="oozie" path="/apps/falcon/extensions/hive-mirroring/resources/runtime/hive-mirroring-workflow.xml"
+    lib="##workflow.lib.path##"/>
     <retry policy="##retry.policy##" delay="##retry.delay##" attempts="3"/>
     <notification type="##notification.type##" to="##notification.receivers##"/>
     <ACL/>

diff --git a/falcon-regression/merlin/src/test/resources/HdfsRecipe/hdfs-replication-template.xml b/falcon-regression/merlin/src/test/resources/HdfsRecipe/hdfs-replication-template.xml
@@ -37,7 +37,7 @@
         <property name="oozie.wf.subworkflow.classpath.inheritance" value="true"/>
     </properties>
 
-    <workflow name="##falcon.recipe.workflow.name##" engine="oozie" path="/apps/data-mirroring/workflows/hdfs-replication-workflow.xml" lib="##workflow.lib.path##"/>
+    <workflow name="##falcon.recipe.workflow.name##" engine="oozie" path="/apps/falcon/extensions/hdfs-mirroring/resources/runtime/hdfs-mirroring-workflow.xml" lib="##workflow.lib.path##"/>
     <retry policy="##falcon.recipe.retry.policy##" delay="##falcon.recipe.retry.delay##" attempts="3"/>
     <notification type="##falcon.recipe.notification.type##" to="##falcon.recipe.notification.receivers##"/>
     <ACL/>

diff --git a/falcon-regression/merlin/src/test/resources/HdfsRecipe/hdfs-replication.properties b/falcon-regression/merlin/src/test/resources/HdfsRecipe/hdfs-replication.properties
@@ -25,7 +25,7 @@ falcon.recipe.name=hdfs-replication-monthly
 ##### Workflow properties
 falcon.recipe.workflow.name=hdfs-dr-workflow
 # Provide Wf absolute path. This can be HDFS or local FS path. If WF is on local FS it will be copied to HDFS
-falcon.recipe.workflow.path=/apps/data-mirroring/workflows/hdfs-replication-workflow.xml
+falcon.recipe.workflow.path=/apps/falcon/extensions/hdfs-mirroring/resources/runtime/hdfs-mirroring-workflow.xml
 
 ##### Cluster properties
 

diff --git a/falcon-regression/merlin/src/test/resources/HiveDrRecipe/hive-disaster-recovery-template.xml b/falcon-regression/merlin/src/test/resources/HiveDrRecipe/hive-disaster-recovery-template.xml
@@ -38,7 +38,7 @@
     </properties>
 
     <workflow name="##workflow.name##" engine="oozie"
-              path="/apps/data-mirroring/workflows/hive-disaster-recovery-workflow.xml" lib="##workflow.lib.path##"/>
+              path="/apps/falcon/extensions/hive-mirroring/resources/runtime/hive-mirroring-workflow.xml" lib="##workflow.lib.path##"/>
     <retry policy="##retry.policy##" delay="##retry.delay##" attempts="3"/>
     <notification type="##falcon.recipe.notification.type##" to="##falcon.recipe.notification.receivers##"/>
     <ACL/>

diff --git a/...n/merlin/src/test/resources/HiveDrSecureRecipe/hive-disaster-recovery-secure-template.xml b/...n/merlin/src/test/resources/HiveDrSecureRecipe/hive-disaster-recovery-secure-template.xml
@@ -38,7 +38,7 @@
     </properties>
 
     <workflow name="##workflow.name##" engine="oozie"
-              path="/apps/data-mirroring/workflows/hive-disaster-recovery-workflow.xml" lib="##workflow.lib.path##"/>
+              path="/apps/falcon/extensions/hive-mirroring/resources/runtime/hive-mirroring-workflow.xml" lib="##workflow.lib.path##"/>
     <retry policy="##retry.policy##" delay="##retry.delay##" attempts="3"/>
     <notification type="##falcon.recipe.notification.type##" to="##falcon.recipe.notification.receivers##"/>
     <ACL/>

diff --git a/falcon-ui/app/js/services/entity/entity-model.js b/falcon-ui/app/js/services/entity/entity-model.js
@@ -302,7 +302,7 @@
           workflow: {
             _name: "hdfs-dr-workflow",
             _engine: "oozie",
-            _path: "/apps/data-mirroring/workflows/hdfs-replication-workflow.xml",
+            _path: "/apps/falcon/extensions/hdfs-mirroring/resources/runtime/hdfs-mirroring-workflow.xml",
             _lib: ""
           },
           retry: {
@@ -434,7 +434,7 @@
           workflow: {
             _name: "falcon-dr-hive-workflow",
             _engine: "oozie",
-            _path: "/apps/data-mirroring/workflows/hive-disaster-recovery-workflow.xml",
+            _path: "/apps/falcon/extensions/hive-mirroring/resources/runtime/hive-mirroring-workflow.xml",
             _lib: ""
           },
           retry: {

diff --git a/src/main/assemblies/distributed-package.xml b/src/main/assemblies/distributed-package.xml
@@ -247,6 +247,16 @@
             <directoryMode>0755</directoryMode>
         </fileSet>
 
+        <fileSet>
+            <directory>../</directory>
+            <outputDirectory>extensions/mirroring</outputDirectory>
+            <excludes>
+                <exclude>*/**</exclude>
+            </excludes>
+            <fileMode>0770</fileMode>
+            <directoryMode>0770</directoryMode>
+        </fileSet>
+
         <fileSet>
             <directory>../examples/app</directory>
             <outputDirectory>examples/app</outputDirectory>

diff --git a/src/main/assemblies/standalone-package.xml b/src/main/assemblies/standalone-package.xml
@@ -270,6 +270,16 @@
             <fileMode>0755</fileMode>
             <directoryMode>0755</directoryMode>
         </fileSet>
+
+        <fileSet>
+            <directory>../</directory>
+            <outputDirectory>extensions/mirroring</outputDirectory>
+            <excludes>
+                <exclude>*/**</exclude>
+            </excludes>
+            <fileMode>0770</fileMode>
+            <directoryMode>0770</directoryMode>
+        </fileSet>
     </fileSets>
 
     <files>

diff --git a/webapp/src/test/resources/hive-mirroring-template.xml b/webapp/src/test/resources/hive-mirroring-template.xml
@@ -38,7 +38,7 @@
     </properties>
 
     <workflow name="##workflow.name##" engine="oozie"
-              path="/apps/data-mirroring/workflows/hive-disaster-recovery-workflow.xml" lib="##workflow.lib.path##"/>
+              path="/apps/falcon/extensions/hive-mirroring/resources/runtime/hive-mirroring-workflow.xml" lib="##workflow.lib.path##"/>
     <retry policy="##retry.policy##" delay="##retry.delay##" attempts="3"/>
     <notification type="##notification.type##" to="##notification.receivers##"/>
     <ACL/>