rebasing from master

pallavi-rao · Aug 5, 2016 · a94d4fe · a94d4fe
1 parent 9e68a57
commit a94d4fe
Show file tree

Hide file tree

Showing 98 changed files with 4,219 additions and 892 deletions.
diff --git a/Installation-steps.txt b/Installation-steps.txt
@@ -41,10 +41,12 @@ a. Building falcon from the source release
 * export MAVEN_OPTS="-Xmx1024m -XX:MaxPermSize=256m -noverify" && mvn clean install
 
 [optionally -Dhadoop.version=<<hadoop.version>> can be appended to build for a specific version of hadoop]
-*Note:* Falcon drops support for Hadoop-1 and only supports Hadoop-2 from Falcon 0.6 onwards
-        Falcon build with JDK 1.7 using -noverify option
-        To compile Falcon with Hive Replication, optionally "-P hadoop-2,hivedr" can be appended. For this
-        Hive >= 1.2.0 and Oozie >= 4.2.0 should be available.
+*Note 1:* Falcon drops support for Hadoop-1 and only supports Hadoop-2 from Falcon 0.6 onwards
+          Falcon build with JDK 1.7 using -noverify option
+*Note 2:* To compile Falcon with addon extensions, append additional profiles to build command using syntax -P<<profile1,profile2>>
+          For Hive Mirroring extension, use profile"hivedr". Hive >= 1.2.0 and Oozie >= 4.2.0 is required
+          For HDFS Snapshot mirroring extension, use profile "hdfs-snapshot-mirroring". Hadoop >= 2.7.0 is required
+          For ADF integration, use profile "adf"
 
 
 b. Building falcon from the source repository
@@ -55,10 +57,12 @@ b. Building falcon from the source repository
 * export MAVEN_OPTS="-Xmx1024m -XX:MaxPermSize=256m -noverify" && mvn clean install
 
 [optionally -Dhadoop.version=<<hadoop.version>> can be appended to build for a specific version of hadoop]
-*Note:* Falcon drops support for Hadoop-1 and only supports Hadoop-2 from Falcon 0.6 onwards
-        Falcon build with JDK 1.7 using -noverify option
-        To compile Falcon with Hive Replication, optionally "-P hadoop-2,hivedr" can be appended. For this
-        Hive >= 1.2.0 and Oozie >= 4.2.0 should be available.
+*Note 1:* Falcon drops support for Hadoop-1 and only supports Hadoop-2 from Falcon 0.6 onwards
+          Falcon build with JDK 1.7 using -noverify option
+*Note 2:* To compile Falcon with addon extensions, append additional profiles to build command using syntax -P<<profile1,profile2>>
+          For Hive Mirroring extension, use profile"hivedr". Hive >= 1.2.0 and Oozie >= 4.2.0 is required
+          For HDFS Snapshot mirroring extension, use profile "hdfs-snapshot-mirroring". Hadoop >= 2.7.0 is required
+          For ADF integration, use profile "adf"
 
 
 2. Deploying Falcon
@@ -118,8 +122,8 @@ c. Using Falcon
 ~~~~~~~~~~~~~~~
 
 * bin/falcon admin -version
-  Falcon server build version: {Version:"0.3-SNAPSHOT-rd7e2be9afa2a5dc96acd1ec9e325f39c6b2f17f7",Mode:"embedded"}
-
+Falcon server build version: {"properties":[{"key":"Version","value":"0.10-rbe02edf0d5b10af27bbac694e536bef30885c00e"},
+{"key":"Mode","value":"embedded"},{"key":"authentication","value":"simple"},{"key":"safemode","value":"false"}]}
 * bin/falcon help
   (for more details about falcon cli usage)
 

diff --git a/NOTICE.txt b/NOTICE.txt
@@ -1,6 +1,6 @@
 Apache Falcon
 
-Copyright 2011-2015 The Apache Software Foundation
+Copyright 2011-2016 The Apache Software Foundation
 
 This product includes software developed at
 The Apache Software Foundation (http://www.apache.org/).
diff --git a/addons/extensions/hdfs-mirroring/src/main/META/hdfs-mirroring-properties.json b/addons/extensions/hdfs-mirroring/src/main/META/hdfs-mirroring-properties.json
@@ -121,6 +121,90 @@
             "description":"Bandwidth in MB for each mapper in DistCP",
             "example":"100"
         },
+        {
+            "propertyName":"tdeEncryptionEnabled",
+            "required":false,
+            "description":"Set this flag to true if TDE encryption is enabled on source and target. Default value is false",
+            "example":"true"
+        },
+        {
+            "propertyName":"overwrite",
+            "required":false,
+            "description":"DitcCP overwrites target files even if they exist at the source, or have the same contents",
+            "example":"true"
+        },
+        {
+            "propertyName":"ignoreErrors",
+            "required":false,
+            "description":"Ignore failures during DistCp",
+            "example":"true"
+        },
+        {
+            "propertyName":"skipChecksum",
+            "required":false,
+            "description":"Skip checksum errors during DistCP",
+            "example":"true"
+        },
+        {
+            "propertyName":"removeDeletedFiles",
+            "required":false,
+            "description":"DistCP deletes the files existing in the dst but not in src",
+            "example":"true"
+        },
+        {
+            "propertyName":"preserveBlockSize",
+            "required":false,
+            "description":"Preserve block size during DistCP",
+            "example":"true"
+        },
+        {
+            "propertyName":"preserveReplicationNumber",
+            "required":false,
+            "description":"Preserve replication number during DistCP",
+            "example":"false"
+        },
+        {
+            "propertyName":"preservePermission",
+            "required":false,
+            "description":"Preserve permission during DistCP",
+            "example":"true"
+        },
+        {
+            "propertyName":"preserveUser",
+            "required":false,
+            "description":"Preserve user during DistCP",
+            "example":"true"
+        },
+        {
+            "propertyName":"preserveGroup",
+            "required":false,
+            "description":"Preserve group during DistCP",
+            "example":"true"
+        },
+        {
+            "propertyName":"preserveChecksumType",
+            "required":false,
+            "description":"Preserve checksum type during DistCP",
+            "example":"true"
+        },
+        {
+            "propertyName":"preserveAcl",
+            "required":false,
+            "description":"Preserve ACL during DistCP",
+            "example":"false"
+        },
+        {
+            "propertyName":"preserveXattr",
+            "required":false,
+            "description":"Preserve Xattr during DistCP",
+            "example":"true"
+        },
+        {
+            "propertyName":"preserveTimes",
+            "required":false,
+            "description":"Preserve access and modification times during DistCP",
+            "example":"true"
+        },
         {
             "propertyName":"jobNotificationType",
             "required":false,

diff --git a/addons/extensions/hdfs-mirroring/src/main/resources/runtime/hdfs-mirroring-workflow.xml b/addons/extensions/hdfs-mirroring/src/main/resources/runtime/hdfs-mirroring-workflow.xml
@@ -63,6 +63,32 @@
             <arg>${distcpMaxMaps}</arg>
             <arg>-mapBandwidth</arg>
             <arg>${distcpMapBandwidth}</arg>
+            <arg>-overwrite</arg>
+            <arg>${overwrite}</arg>
+            <arg>-ignoreErrors</arg>
+            <arg>${ignoreErrors}</arg>
+            <arg>-skipChecksum</arg>
+            <arg>${skipChecksum}</arg>
+            <arg>-removeDeletedFiles</arg>
+            <arg>${removeDeletedFiles}</arg>
+            <arg>-preserveBlockSize</arg>
+            <arg>${preserveBlockSize}</arg>
+            <arg>-preserveReplicationNumber</arg>
+            <arg>${preserveReplicationNumber}</arg>
+            <arg>-preservePermission</arg>
+            <arg>${preservePermission}</arg>
+            <arg>-preserveUser</arg>
+            <arg>${preserveUser}</arg>
+            <arg>-preserveGroup</arg>
+            <arg>${preserveGroup}</arg>
+            <arg>-preserveChecksumType</arg>
+            <arg>${preserveChecksumType}</arg>
+            <arg>-preserveAcl</arg>
+            <arg>${preserveAcl}</arg>
+            <arg>-preserveXattr</arg>
+            <arg>${preserveXattr}</arg>
+            <arg>-preserveTimes</arg>
+            <arg>${preserveTimes}</arg>
             <arg>-sourcePaths</arg>
             <arg>${sourceDir}</arg>
             <arg>-targetPath</arg>
@@ -71,6 +97,8 @@
             <arg>FILESYSTEM</arg>
             <arg>-availabilityFlag</arg>
             <arg>${availabilityFlag == 'NA' ? "NA" : availabilityFlag}</arg>
+            <arg>-tdeEncryptionEnabled</arg>
+            <arg>${tdeEncryptionEnabled}</arg>
             <arg>-counterLogDir</arg>
             <arg>${logDir}/job-${nominalTime}/${srcClusterName == 'NA' ? '' : srcClusterName}</arg>
         </java>

diff --git a/.../extensions/hdfs-snapshot-mirroring/src/main/META/hdfs-snapshot-mirroring-properties.json b/.../extensions/hdfs-snapshot-mirroring/src/main/META/hdfs-snapshot-mirroring-properties.json
@@ -140,13 +140,13 @@
       "example":"10"
     },
     {
-      "propertyName":"distcpMaxMaps",
+      "propertyName":"maxMaps",
       "required":false,
       "description":"Maximum number of mappers for DistCP",
       "example":"1"
     },
     {
-      "propertyName":"distcpMapBandwidth",
+      "propertyName":"mapBandwidth",
       "required":false,
       "description":"Bandwidth in MB for each mapper in DistCP",
       "example":"100"
@@ -157,6 +157,78 @@
       "description":"Specify if TDE based encryption is enabled on source and target dirs",
       "example":"false"
     },
+    {
+      "propertyName":"ignoreErrors",
+      "required":false,
+      "description":"Ignore failures during DistCp",
+      "example":"true"
+    },
+    {
+      "propertyName":"skipChecksum",
+      "required":false,
+      "description":"Skip checksum errors during DistCP",
+      "example":"true"
+    },
+    {
+      "propertyName":"removeDeletedFiles",
+      "required":false,
+      "description":"DistCP deletes the files existing in the dst but not in src",
+      "example":"true"
+    },
+    {
+      "propertyName":"preserveBlockSize",
+      "required":false,
+      "description":"Preserve block size during DistCP",
+      "example":"true"
+    },
+    {
+      "propertyName":"preserveReplicationNumber",
+      "required":false,
+      "description":"Preserve replication number during DistCP",
+      "example":"false"
+    },
+    {
+      "propertyName":"preservePermission",
+      "required":false,
+      "description":"Preserve permission during DistCP",
+      "example":"true"
+    },
+    {
+      "propertyName":"preserveUser",
+      "required":false,
+      "description":"Preserve user during DistCP",
+      "example":"true"
+    },
+    {
+      "propertyName":"preserveGroup",
+      "required":false,
+      "description":"Preserve group during DistCP",
+      "example":"true"
+    },
+    {
+      "propertyName":"preserveChecksumType",
+      "required":false,
+      "description":"Preserve checksum type during DistCP",
+      "example":"true"
+    },
+    {
+      "propertyName":"preserveAcl",
+      "required":false,
+      "description":"Preserve ACL during DistCP",
+      "example":"false"
+    },
+    {
+      "propertyName":"preserveXattr",
+      "required":false,
+      "description":"Preserve Xattr during DistCP",
+      "example":"true"
+    },
+    {
+      "propertyName":"preserveTimes",
+      "required":false,
+      "description":"Preserve access and modification times during DistCP",
+      "example":"true"
+    },
     {
       "propertyName":"jobNotificationType",
       "required":false,
@@ -170,4 +242,4 @@
       "example":"user1@gmail.com, user2@gmail.com"
     }
   ]
-}
+}
diff --git a/...s/hdfs-snapshot-mirroring/src/main/resources/runtime/hdfs-snapshot-mirroring-workflow.xml b/...s/hdfs-snapshot-mirroring/src/main/resources/runtime/hdfs-snapshot-mirroring-workflow.xml
@@ -59,10 +59,36 @@
             <main-class>org.apache.falcon.snapshots.replication.HdfsSnapshotReplicator</main-class>
             <arg>-Dmapred.job.queue.name=${queueName}</arg>
             <arg>-Dmapred.job.priority=${jobPriority}</arg>
-            <arg>-distcpMaxMaps</arg>
-            <arg>${distcpMaxMaps}</arg>
-            <arg>-distcpMapBandwidth</arg>
-            <arg>${distcpMapBandwidth}</arg>
+            <arg>-maxMaps</arg>
+            <arg>${maxMaps}</arg>
+            <arg>-mapBandwidth</arg>
+            <arg>${mapBandwidth}</arg>
+            <arg>-overwrite</arg>
+            <arg>${overwrite}</arg>
+            <arg>-ignoreErrors</arg>
+            <arg>${ignoreErrors}</arg>
+            <arg>-skipChecksum</arg>
+            <arg>${skipChecksum}</arg>
+            <arg>-removeDeletedFiles</arg>
+            <arg>${removeDeletedFiles}</arg>
+            <arg>-preserveBlockSize</arg>
+            <arg>${preserveBlockSize}</arg>
+            <arg>-preserveReplicationNumber</arg>
+            <arg>${preserveReplicationNumber}</arg>
+            <arg>-preservePermission</arg>
+            <arg>${preservePermission}</arg>
+            <arg>-preserveUser</arg>
+            <arg>${preserveUser}</arg>
+            <arg>-preserveGroup</arg>
+            <arg>${preserveGroup}</arg>
+            <arg>-preserveChecksumType</arg>
+            <arg>${preserveChecksumType}</arg>
+            <arg>-preserveAcl</arg>
+            <arg>${preserveAcl}</arg>
+            <arg>-preserveXattr</arg>
+            <arg>${preserveXattr}</arg>
+            <arg>-preserveTimes</arg>
+            <arg>${preserveTimes}</arg>
             <arg>-sourceNN</arg>
             <arg>${sourceNN}</arg>
             <arg>-sourceExecUrl</arg>
@@ -169,4 +195,4 @@
         </message>
     </kill>
     <end name="end"/>
-</workflow-app>
+</workflow-app>
diff --git a/addons/extensions/hive-mirroring/src/main/META/hive-mirroring-properties.json b/addons/extensions/hive-mirroring/src/main/META/hive-mirroring-properties.json
@@ -94,9 +94,15 @@
         {
             "propertyName":"sourceHiveServer2Uri",
             "required":true,
-            "description":"Hive2 server end point",
+            "description":"Hive2 server end point. If Zookeeper discovery mode is enabled zookeeper_ensemble is expected",
             "example":"hive2://localhost:10000"
         },
+        {
+            "propertyName":"sourceHiveServer2ExtraOpts",
+            "required":false,
+            "description":"Extra opts required when SSL is enbaled, Http mode and when zookeeper discovery is used",
+            "example":"serviceDiscoveryMode=zooKeeper; zooKeeperNamespace=<hiveserver2_namespace>"
+        },
         {
             "propertyName":"sourceDatabases",
             "required":true,
@@ -130,9 +136,15 @@
         {
             "propertyName":"targetHiveServer2Uri",
             "required":true,
-            "description":"Hive2 server end point",
+            "description":"Hive2 server end point. If Zookeeper discovery mode is enabled zookeeper_ensemble is expected",
             "example":"hive2://localhost:10000"
         },
+        {
+            "propertyName":"targetHiveServer2ExtraOpts",
+            "required":false,
+            "description":"Extra opts required when SSL is enbaled, Http mode and when zookeeper discovery is used",
+            "example":"serviceDiscoveryMode=zooKeeper; zooKeeperNamespace=<hiveserver2_namespace>"
+        },
         {
             "propertyName":"targetStagingPath",
             "required":false,