Merge pull request #12750 from bgsanthosh/COMPX-8551

Compx 8551 - Default configs for parallel renames and rename resiliency
hortonworks · May 13, 2022 · 7925c32 · 7925c32
2 parents 8841d5d + 9fa3aa5
commit 7925c32
Show file tree

Hide file tree

Showing 9 changed files with 9 additions and 9 deletions.
diff --git a/core/src/main/resources/defaults/blueprints/7.2.14/cdp-data-engineering-ha.bp b/core/src/main/resources/defaults/blueprints/7.2.14/cdp-data-engineering-ha.bp
@@ -34,7 +34,7 @@
           },
           {
             "name": "core_site_safety_valve",
-            "value": "<property><name>fs.s3a.buffer.dir</name><value>${env.LOCAL_DIRS:-${hadoop.tmp.dir}}/s3a</value></property><property><name>fs.s3a.committer.name</name><value>directory</value></property><property><name>fs.azure.buffer.dir</name><value>${env.LOCAL_DIRS:-${hadoop.tmp.dir}}/abfs</value></property>"
+            "value": "<property><name>fs.s3a.buffer.dir</name><value>${env.LOCAL_DIRS:-${hadoop.tmp.dir}}/s3a</value></property><property><name>fs.s3a.committer.name</name><value>directory</value></property><property><name>fs.azure.buffer.dir</name><value>${env.LOCAL_DIRS:-${hadoop.tmp.dir}}/abfs</value></property><property><name>hadoop.mapreduce.fileoutputcommitter.algorithm.version.v1.experimental.parallel.rename.recovery</name><value>true</value></property><property><name>hadoop.mapreduce.fileoutputcommitter.algorithm.version.v1.experimental.parallel.task.commit</name><value>true</value></property><property><name>hadoop.mapreduce.fileoutputcommitter.algorithm.version.v1.experimental.mv.threads</name><value>16</value></property>"
           }
         ],
         "roleConfigGroups": [

diff --git a/core/src/main/resources/defaults/blueprints/7.2.14/cdp-data-engineering-spark3.bp b/core/src/main/resources/defaults/blueprints/7.2.14/cdp-data-engineering-spark3.bp
@@ -26,7 +26,7 @@
           },
           {
             "name": "core_site_safety_valve",
-            "value": "<property><name>fs.s3a.buffer.dir</name><value>${env.LOCAL_DIRS:-${hadoop.tmp.dir}}/s3a</value></property><property><name>fs.s3a.committer.name</name><value>directory</value></property><property><name>fs.azure.buffer.dir</name><value>${env.LOCAL_DIRS:-${hadoop.tmp.dir}}/abfs</value></property>"
+            "value": "<property><name>fs.s3a.buffer.dir</name><value>${env.LOCAL_DIRS:-${hadoop.tmp.dir}}/s3a</value></property><property><name>fs.s3a.committer.name</name><value>directory</value></property><property><name>fs.azure.buffer.dir</name><value>${env.LOCAL_DIRS:-${hadoop.tmp.dir}}/abfs</value></property><property><name>hadoop.mapreduce.fileoutputcommitter.algorithm.version.v1.experimental.parallel.rename.recovery</name><value>true</value></property><property><name>hadoop.mapreduce.fileoutputcommitter.algorithm.version.v1.experimental.parallel.task.commit</name><value>true</value></property><property><name>hadoop.mapreduce.fileoutputcommitter.algorithm.version.v1.experimental.mv.threads</name><value>16</value></property>"
           }
         ],
         "roleConfigGroups": [

diff --git a/core/src/main/resources/defaults/blueprints/7.2.14/cdp-data-engineering.bp b/core/src/main/resources/defaults/blueprints/7.2.14/cdp-data-engineering.bp
@@ -32,7 +32,7 @@
           },
           {
             "name": "core_site_safety_valve",
-            "value": "<property><name>fs.s3a.buffer.dir</name><value>${env.LOCAL_DIRS:-${hadoop.tmp.dir}}/s3a</value></property><property><name>fs.s3a.committer.name</name><value>directory</value></property><property><name>fs.azure.buffer.dir</name><value>${env.LOCAL_DIRS:-${hadoop.tmp.dir}}/abfs</value></property>"
+            "value": "<property><name>fs.s3a.buffer.dir</name><value>${env.LOCAL_DIRS:-${hadoop.tmp.dir}}/s3a</value></property><property><name>fs.s3a.committer.name</name><value>directory</value></property><property><name>fs.azure.buffer.dir</name><value>${env.LOCAL_DIRS:-${hadoop.tmp.dir}}/abfs</value></property><property><name>hadoop.mapreduce.fileoutputcommitter.algorithm.version.v1.experimental.parallel.rename.recovery</name><value>true</value></property><property><name>hadoop.mapreduce.fileoutputcommitter.algorithm.version.v1.experimental.parallel.task.commit</name><value>true</value></property><property><name>hadoop.mapreduce.fileoutputcommitter.algorithm.version.v1.experimental.mv.threads</name><value>16</value></property>"
           }
         ],
         "roleConfigGroups": [

diff --git a/core/src/main/resources/defaults/blueprints/7.2.15/cdp-data-engineering-ha.bp b/core/src/main/resources/defaults/blueprints/7.2.15/cdp-data-engineering-ha.bp
@@ -34,7 +34,7 @@
           },
           {
             "name": "core_site_safety_valve",
-            "value": "<property><name>fs.s3a.buffer.dir</name><value>${env.LOCAL_DIRS:-${hadoop.tmp.dir}}/s3a</value></property><property><name>fs.s3a.committer.name</name><value>directory</value></property><property><name>fs.azure.buffer.dir</name><value>${env.LOCAL_DIRS:-${hadoop.tmp.dir}}/abfs</value></property>"
+            "value": "<property><name>fs.s3a.buffer.dir</name><value>${env.LOCAL_DIRS:-${hadoop.tmp.dir}}/s3a</value></property><property><name>fs.s3a.committer.name</name><value>directory</value></property><property><name>fs.azure.buffer.dir</name><value>${env.LOCAL_DIRS:-${hadoop.tmp.dir}}/abfs</value></property><property><name>hadoop.mapreduce.fileoutputcommitter.algorithm.version.v1.experimental.parallel.rename.recovery</name><value>true</value></property><property><name>hadoop.mapreduce.fileoutputcommitter.algorithm.version.v1.experimental.parallel.task.commit</name><value>true</value></property><property><name>hadoop.mapreduce.fileoutputcommitter.algorithm.version.v1.experimental.mv.threads</name><value>16</value></property>"
           }
         ],
         "roleConfigGroups": [

diff --git a/core/src/main/resources/defaults/blueprints/7.2.15/cdp-data-engineering-spark3.bp b/core/src/main/resources/defaults/blueprints/7.2.15/cdp-data-engineering-spark3.bp
@@ -26,7 +26,7 @@
           },
           {
             "name": "core_site_safety_valve",
-            "value": "<property><name>fs.s3a.buffer.dir</name><value>${env.LOCAL_DIRS:-${hadoop.tmp.dir}}/s3a</value></property><property><name>fs.s3a.committer.name</name><value>directory</value></property><property><name>fs.azure.buffer.dir</name><value>${env.LOCAL_DIRS:-${hadoop.tmp.dir}}/abfs</value></property>"
+            "value": "<property><name>fs.s3a.buffer.dir</name><value>${env.LOCAL_DIRS:-${hadoop.tmp.dir}}/s3a</value></property><property><name>fs.s3a.committer.name</name><value>directory</value></property><property><name>fs.azure.buffer.dir</name><value>${env.LOCAL_DIRS:-${hadoop.tmp.dir}}/abfs</value></property><property><name>hadoop.mapreduce.fileoutputcommitter.algorithm.version.v1.experimental.parallel.rename.recovery</name><value>true</value></property><property><name>hadoop.mapreduce.fileoutputcommitter.algorithm.version.v1.experimental.parallel.task.commit</name><value>true</value></property><property><name>hadoop.mapreduce.fileoutputcommitter.algorithm.version.v1.experimental.mv.threads</name><value>16</value></property>"
           }
         ],
         "roleConfigGroups": [

diff --git a/core/src/main/resources/defaults/blueprints/7.2.15/cdp-data-engineering.bp b/core/src/main/resources/defaults/blueprints/7.2.15/cdp-data-engineering.bp
@@ -32,7 +32,7 @@
           },
           {
             "name": "core_site_safety_valve",
-            "value": "<property><name>fs.s3a.buffer.dir</name><value>${env.LOCAL_DIRS:-${hadoop.tmp.dir}}/s3a</value></property><property><name>fs.s3a.committer.name</name><value>directory</value></property><property><name>fs.azure.buffer.dir</name><value>${env.LOCAL_DIRS:-${hadoop.tmp.dir}}/abfs</value></property>"
+            "value": "<property><name>fs.s3a.buffer.dir</name><value>${env.LOCAL_DIRS:-${hadoop.tmp.dir}}/s3a</value></property><property><name>fs.s3a.committer.name</name><value>directory</value></property><property><name>fs.azure.buffer.dir</name><value>${env.LOCAL_DIRS:-${hadoop.tmp.dir}}/abfs</value></property><property><name>hadoop.mapreduce.fileoutputcommitter.algorithm.version.v1.experimental.parallel.rename.recovery</name><value>true</value></property><property><name>hadoop.mapreduce.fileoutputcommitter.algorithm.version.v1.experimental.parallel.task.commit</name><value>true</value></property><property><name>hadoop.mapreduce.fileoutputcommitter.algorithm.version.v1.experimental.mv.threads</name><value>16</value></property>"
           }
         ],
         "roleConfigGroups": [

diff --git a/core/src/main/resources/defaults/blueprints/7.2.16/cdp-data-engineering-ha.bp b/core/src/main/resources/defaults/blueprints/7.2.16/cdp-data-engineering-ha.bp
@@ -34,7 +34,7 @@
           },
           {
             "name": "core_site_safety_valve",
-            "value": "<property><name>fs.s3a.buffer.dir</name><value>${env.LOCAL_DIRS:-${hadoop.tmp.dir}}/s3a</value></property><property><name>fs.s3a.committer.name</name><value>directory</value></property><property><name>fs.azure.buffer.dir</name><value>${env.LOCAL_DIRS:-${hadoop.tmp.dir}}/abfs</value></property>"
+            "value": "<property><name>fs.s3a.buffer.dir</name><value>${env.LOCAL_DIRS:-${hadoop.tmp.dir}}/s3a</value></property><property><name>fs.s3a.committer.name</name><value>directory</value></property><property><name>fs.azure.buffer.dir</name><value>${env.LOCAL_DIRS:-${hadoop.tmp.dir}}/abfs</value></property><property><name>hadoop.mapreduce.fileoutputcommitter.algorithm.version.v1.experimental.parallel.rename.recovery</name><value>true</value></property><property><name>hadoop.mapreduce.fileoutputcommitter.algorithm.version.v1.experimental.parallel.task.commit</name><value>true</value></property><property><name>hadoop.mapreduce.fileoutputcommitter.algorithm.version.v1.experimental.mv.threads</name><value>16</value></property>"
           }
         ],
         "roleConfigGroups": [

diff --git a/core/src/main/resources/defaults/blueprints/7.2.16/cdp-data-engineering-spark3.bp b/core/src/main/resources/defaults/blueprints/7.2.16/cdp-data-engineering-spark3.bp
@@ -26,7 +26,7 @@
           },
           {
             "name": "core_site_safety_valve",
-            "value": "<property><name>fs.s3a.buffer.dir</name><value>${env.LOCAL_DIRS:-${hadoop.tmp.dir}}/s3a</value></property><property><name>fs.s3a.committer.name</name><value>directory</value></property><property><name>fs.azure.buffer.dir</name><value>${env.LOCAL_DIRS:-${hadoop.tmp.dir}}/abfs</value></property>"
+            "value": "<property><name>fs.s3a.buffer.dir</name><value>${env.LOCAL_DIRS:-${hadoop.tmp.dir}}/s3a</value></property><property><name>fs.s3a.committer.name</name><value>directory</value></property><property><name>fs.azure.buffer.dir</name><value>${env.LOCAL_DIRS:-${hadoop.tmp.dir}}/abfs</value></property><property><name>hadoop.mapreduce.fileoutputcommitter.algorithm.version.v1.experimental.parallel.rename.recovery</name><value>true</value></property><property><name>hadoop.mapreduce.fileoutputcommitter.algorithm.version.v1.experimental.parallel.task.commit</name><value>true</value></property><property><name>hadoop.mapreduce.fileoutputcommitter.algorithm.version.v1.experimental.mv.threads</name><value>16</value></property>"
           }
         ],
         "roleConfigGroups": [

diff --git a/core/src/main/resources/defaults/blueprints/7.2.16/cdp-data-engineering.bp b/core/src/main/resources/defaults/blueprints/7.2.16/cdp-data-engineering.bp
@@ -32,7 +32,7 @@
           },
           {
             "name": "core_site_safety_valve",
-            "value": "<property><name>fs.s3a.buffer.dir</name><value>${env.LOCAL_DIRS:-${hadoop.tmp.dir}}/s3a</value></property><property><name>fs.s3a.committer.name</name><value>directory</value></property><property><name>fs.azure.buffer.dir</name><value>${env.LOCAL_DIRS:-${hadoop.tmp.dir}}/abfs</value></property>"
+            "value": "<property><name>fs.s3a.buffer.dir</name><value>${env.LOCAL_DIRS:-${hadoop.tmp.dir}}/s3a</value></property><property><name>fs.s3a.committer.name</name><value>directory</value></property><property><name>fs.azure.buffer.dir</name><value>${env.LOCAL_DIRS:-${hadoop.tmp.dir}}/abfs</value></property><property><name>hadoop.mapreduce.fileoutputcommitter.algorithm.version.v1.experimental.parallel.rename.recovery</name><value>true</value></property><property><name>hadoop.mapreduce.fileoutputcommitter.algorithm.version.v1.experimental.parallel.task.commit</name><value>true</value></property><property><name>hadoop.mapreduce.fileoutputcommitter.algorithm.version.v1.experimental.mv.threads</name><value>16</value></property>"
           }
         ],
         "roleConfigGroups": [