Merge [TRAFODION-2697] Set bulkload permissions for hbase upload

apache · Aug 2, 2017 · d86a0ba · d86a0ba
2 parents ed8666d + a9af698
commit d86a0ba
Show file tree

Hide file tree

Showing 7 changed files with 39 additions and 10 deletions.
diff --git a/core/sql/regress/hive/EXPECTED018 b/core/sql/regress/hive/EXPECTED018
@@ -530,7 +530,7 @@ cat /tmp/merged_customer_address | wc -l
 >>--------------------------
 >>--exp2
 >>explain options 'f' 
-+>UNLOAD EXTRACT TO '/bulkload/customer_demographics'
++>UNLOAD EXTRACT TO '/user/trafodion/bulkload/customer_demographics'
 +>select * from trafodion.hbase.customer_demographics 
 +><<+ cardinality 10e10 >>;
 
@@ -614,7 +614,7 @@ LC   RC   OP   OPERATOR              OPT       DESCRIPTION           CARD
 +>WITH PURGEDATA FROM TARGET
 +>MERGE FILE  'merged_customer_demogs_3' OVERWRITE
 +>--COMPRESSION GZIP
-+>INTO '/bulkload/customer_demographics_salt'
++>INTO '/user/trafodion/bulkload/customer_demographics_salt'
 +>select * from trafodion.hbase.customer_demographics_salt 
 +><<+ cardinality 10e10 >>;
 Task: UNLOAD           Status: Started

diff --git a/core/sql/regress/hive/TEST018 b/core/sql/regress/hive/TEST018
@@ -308,7 +308,7 @@ log LOG018;
 --------------------------
 --exp2
 explain options 'f' 
-UNLOAD EXTRACT TO '/bulkload/customer_demographics'
+UNLOAD EXTRACT TO '/user/trafodion/bulkload/customer_demographics'
 select * from trafodion.hbase.customer_demographics 
 <<+ cardinality 10e10 >>;
 --unload 2
@@ -351,7 +351,7 @@ UNLOAD
 WITH PURGEDATA FROM TARGET
 MERGE FILE  'merged_customer_demogs_3' OVERWRITE
 --COMPRESSION GZIP
-INTO '/bulkload/customer_demographics_salt'
+INTO '/user/trafodion/bulkload/customer_demographics_salt'
 select * from trafodion.hbase.customer_demographics_salt 
 <<+ cardinality 10e10 >>;
 

diff --git a/core/trafodion.spec b/core/trafodion.spec
@@ -96,9 +96,19 @@ then
 fi
 if ! getent passwd trafodion > /dev/null
 then
-  /usr/sbin/useradd --shell /bin/bash -m trafodion -g trafodion -G hbase,hive --home /home/trafodion > /dev/null 2>&1
-else
-  /usr/sbin/usermod -a -G hbase,hive trafodion > /dev/null 2>&1
+  /usr/sbin/useradd --shell /bin/bash -m trafodion -g trafodion --home /home/trafodion > /dev/null 2>&1
+fi
+if getent group hbase > /dev/null
+then
+  /usr/sbin/usermod -a -G hbase trafodion > /dev/null 2>&1
+fi
+if getent group hive > /dev/null
+then
+  /usr/sbin/usermod -a -G hive trafodion > /dev/null 2>&1
+fi
+if getent group hadoop > /dev/null
+then
+  /usr/sbin/usermod -a -G hadoop trafodion > /dev/null 2>&1
 fi
 chmod go+rx /home/trafodion
 

diff --git a/install/ambari-installer/traf-mpack/common-services/TRAFODION/2.1/package/scripts/params.py b/install/ambari-installer/traf-mpack/common-services/TRAFODION/2.1/package/scripts/params.py
@@ -54,6 +54,7 @@
 traf_user = 'trafodion'
 traf_group = 'trafodion'
 hdfs_user = config['configurations']['hadoop-env']['hdfs_user']
+user_group = config['configurations']['cluster-env']['user_group']
 hbase_user = config['configurations']['hbase-env']['hbase_user']
 hbase_staging = config['configurations']['hbase-site']['hbase.bulkload.staging.dir']
 

diff --git a/...ari-installer/traf-mpack/common-services/TRAFODION/2.1/package/scripts/trafodionmaster.py b/...ari-installer/traf-mpack/common-services/TRAFODION/2.1/package/scripts/trafodionmaster.py
@@ -131,6 +131,12 @@ def start(self, env):
                          owner=params.hbase_user,
                          group=params.hbase_user,
                         )
+    params.HdfsDirectory("/user/trafodion",
+                         action="create_on_execute",
+                         owner=params.traf_user,
+                         group=params.traf_group,
+                         mode=0755,
+                        )
     params.HdfsDirectory("/user/trafodion/trafodion_backups",
                          action="create_on_execute",
                          owner=params.traf_user,
@@ -139,7 +145,8 @@ def start(self, env):
     params.HdfsDirectory("/user/trafodion/bulkload",
                          action="create_on_execute",
                          owner=params.traf_user,
-                         group=params.traf_group,
+                         group=params.user_group,
+                         mode=0750,
                         )
     params.HdfsDirectory("/user/trafodion/lobs",
                          action="create_on_execute",

diff --git a/install/python-installer/scripts/hdfs_cmds.py b/install/python-installer/scripts/hdfs_cmds.py
@@ -45,11 +45,15 @@ def run():
     traf_user = dbcfgs['traf_user']
     hdfs_user = dbcfgs['hdfs_user']
     hbase_user = dbcfgs['hbase_user']
+    hbase_group = cmd_output('%s groups %s | cut -d" " -f3' % (hdfs_bin, hbase_user))
 
     run_cmd_as_user(hdfs_user, '%s dfsadmin -safemode wait' % hdfs_bin)
-    run_cmd_as_user(hdfs_user, '%s dfs -mkdir -p %s/{trafodion_backups,bulkload,lobs} /bulkload /lobs /hbase/archive' % (hdfs_bin, traf_loc))
+    run_cmd_as_user(hdfs_user, '%s dfs -mkdir -p %s/{trafodion_backups,bulkload,lobs} /hbase/archive' % (hdfs_bin, traf_loc))
     run_cmd_as_user(hdfs_user, '%s dfs -chown -R %s:%s /hbase/archive' % (hdfs_bin, hbase_user, hbase_user))
-    run_cmd_as_user(hdfs_user, '%s dfs -chown -R %s:%s %s %s/{trafodion_backups,bulkload,lobs} /bulkload /lobs' % (hdfs_bin, traf_user, traf_user, traf_loc, traf_loc))
+    run_cmd_as_user(hdfs_user, '%s dfs -chown -R %s:%s %s %s/{trafodion_backups,bulkload,lobs}' % (hdfs_bin, traf_user, traf_user, traf_loc, traf_loc))
+    run_cmd_as_user(hdfs_user, '%s dfs -chmod 0755 %s' % (hdfs_bin, traf_loc))
+    run_cmd_as_user(hdfs_user, '%s dfs -chmod 0750 %s/{trafodion_backups,bulkload,lobs}' % (hdfs_bin, traf_loc))
+    run_cmd_as_user(hdfs_user, '%s dfs -chgrp %s %s/bulkload' % (hdfs_bin, hbase_group, traf_loc))
     run_cmd_as_user(hdfs_user, '%s dfs -setfacl -R -m user:%s:rwx /hbase/archive' % (hdfs_bin, traf_user))
     run_cmd_as_user(hdfs_user, '%s dfs -setfacl -R -m default:user:%s:rwx /hbase/archive' % (hdfs_bin, traf_user))
     run_cmd_as_user(hdfs_user, '%s dfs -setfacl -R -m mask::rwx /hbase/archive' % hdfs_bin)

diff --git a/install/python-installer/scripts/traf_user.py b/install/python-installer/scripts/traf_user.py
@@ -69,6 +69,13 @@ def run():
             run_cmd('groupadd %s > /dev/null 2>&1' % traf_group)
         traf_pwd = dbcfgs['traf_pwd']
         run_cmd('useradd --shell /bin/bash -m %s -g %s --home %s --password "$(openssl passwd %s)"' % (traf_user, traf_group, traf_user_dir, traf_pwd))
+    # hbase group is generally either hbase or hadoop, depending on distro
+    if cmd_output('getent group hbase'):
+        cmd_output('/usr/sbin/usermod -a -G hbase %s' % traf_user)
+    if cmd_output('getent group hadoop'):
+        cmd_output('/usr/sbin/usermod -a -G hadoop %s' % traf_user)
+    if cmd_output('getent group hive'):
+        cmd_output('/usr/sbin/usermod -a -G hive %s' % traf_user)
 
     if not os.path.exists(traf_user_dir):
         run_cmd('mkdir -p %s' % traf_user_dir)