Merge pull request #70 from FINRAOS/1.3.0

1.3.0

.travis.yml

@@ -85,7 +85,7 @@ deploy:
     body: Automated herd-mdl (metastor) build from $TRAVIS_BRANCH ($TRAVIS_COMMIT) built by Travis CI on $(date +'%F %T %Z').
 
   - provider: script
-    script: bash scripts/publish_docker.sh
+    script: bash $TRAVIS_BUILD_DIR/scripts/publish_docker.sh ${component_version} $TRAVIS_BUILD_DIR/mdl/build/
     on:
       repo: FINRAOS/herd-mdl
       branch: master

bdsql/pom.xml

@@ -21,7 +21,7 @@
 
   <groupId>org.finra.herd-mdl</groupId>
   <artifactId>bdsql</artifactId>
-  <version>1.1.0</version>
+  <version>1.2.0</version>
 
   <licenses>
     <license>

bdsql/src/main/conf/lambda_payload.json

@@ -0,0 +1,9 @@
+{
+  "Records": [
+    {
+      "Sns": {
+        "Message": "{\"action\":\"full_sync\"}"
+      }
+    }
+  ]
+}

bdsql/src/main/scripts/configureBDSQL.sh

@@ -41,6 +41,9 @@ function execute_cmd {
 
 execute_cmd "cd /home/hadoop"
 
+# configure region
+execute_cmd "aws configure set default.region ${region}"
+
 # Register the target instance to the ELB
 targetGroupArn=`aws elbv2 describe-target-groups --region ${region} --names ${mdlInstanceName}-BdsqlTargetGroup | grep TargetGroupArn | cut -d":" -f2- | tr -d '\"' | tr -d ',' | tr -d ' '`
 instanceId=`curl http://169.254.169.254/latest/meta-data/instance-id`
@@ -91,11 +94,22 @@ done
 if [ "${enableSSLAndAuth}" = "true" ] ; then
     # Change the role_map
     export metastorDBPassword=$(aws ssm get-parameter --name /app/MDL/${mdlInstanceName}/${environment}/METASTOR/RDS/hiveAccount --with-decryption --region ${region} --output text --query Parameter.Value)
-    mysql -h ${metastorDBHost} -u ${metastorDBUser} -p${metastorDBPassword} -D metastor -e " INSERT INTO ROLE_MAP VALUES (101,UNIX_TIMESTAMP(),1,'hive','USER','ldap_admin_user','USER',1); ;"
+    admin_user=$(aws ssm get-parameter --name /app/MDL/${mdlInstanceName}/${environment}/LDAP/User/HerdAdminUsername --output text --query Parameter.Value)
+
+    mysql -h ${metastorDBHost} -u ${metastorDBUser} -p${metastorDBPassword} -D metastor -e "INSERT INTO ROLE_MAP VALUES (101,UNIX_TIMESTAMP(),1,'hive','USER','${admin_user}','USER',1); ;"
 
+    ### Note: This is no longer needed since we are now driving authorizations off of user-namespace permissions
+    ### To switch back to LDAP-driven Hive authorizations - uncomment the following lines and disable the lambda trigger which follows
     # Execute authorization codebase
-    execute_cmd "sudo scripts/sql_auth.sh"
-    execute_cmd "sudo aws s3 cp scripts/sql_auth.sh s3://${mdlStagingBucketName}/BDSQL/sql_auth.sh"
+    # execute_cmd "sudo scripts/sql_auth.sh"
+    # execute_cmd "sudo aws s3 cp scripts/sql_auth.sh s3://${mdlStagingBucketName}/BDSQL/sql_auth.sh"
+
+    ### Invoke user-namespace authorization lambda
+    # get the user-namespace auth lambda's ARN
+    lambda_arn=$(aws ssm get-parameter --name /app/MDL/${mdlInstanceName}/${environment}/Resources/Lambda/NsAuthSyncArn --region ${region} --output text --query Parameter.Value)
+    # invoke the lambda and pass it a payload which makes it run a full_sync
+    execute_cmd "aws lambda invoke --function-name ${lambda_arn} --invocation-type RequestResponse --log-type None --payload file:///home/hadoop/conf/lambda_payload.json lambda_outfile.txt"
+
 fi
 
 echo "Everything looks good"

bdsql/src/main/scripts/smokeTesting.sh

@@ -26,7 +26,7 @@ function log(){
     echo "$(date '+%F %T') bdsql-$LOG_SOURCE[$$]: $MSG"
 }
 
-function run_preto_query(){
+function run_presto_query(){
 
     local APP_USER="$1"
     local QUERY="$2"
@@ -57,8 +57,8 @@ function run_preto_query(){
 function cleanup_tmp_objects(){
 
     log "removing tmp objects"
-    run_preto_query "$APP_USER" "SHOW TABLES" | sed -e 's/"//g' | egrep '^t[0-9]+(_|$)' | while read t; do
-        run_preto_query "$APP_USER" "DROP TABLE user_$APP_USER.$t"
+    run_presto_query "$APP_USER" "SHOW TABLES" | sed -e 's/"//g' | egrep '^t[0-9]+(_|$)' | while read t; do
+    run_presto_query "$APP_USER" "DROP TABLE user_$APP_USER.$t"
     done
 
 }
@@ -77,28 +77,28 @@ function main(){
     cleanup_tmp_objects
     echo
 
-    run_preto_query "$APP_USER" "select * from system.runtime.nodes"
-    run_preto_query "$APP_USER" "select * from system.metadata.catalogs"
+    run_presto_query "$APP_USER" "select * from system.runtime.nodes"
+    run_presto_query "$APP_USER" "select * from system.metadata.catalogs"
     echo
 
-    run_preto_query "$APP_USER" "select current_timestamp"
+    run_presto_query "$APP_USER" "select current_timestamp"
     echo
 
-    run_preto_query "$APP_USER" "show schemas"
-    run_preto_query "$APP_USER" "show tables"
+    run_presto_query "$APP_USER" "show schemas"
+    run_presto_query "$APP_USER" "show tables"
     echo
 
-    run_preto_query "$APP_USER" "create table t0_default (my_id bigint, my_string varchar)"
-    run_preto_query "$APP_USER" "insert into t0_default values (1,'abc')"
-    run_preto_query "$APP_USER" "select * from t0_default"
+    run_presto_query "$APP_USER" "create table t0_default (my_id bigint, my_string varchar)"
+    run_presto_query "$APP_USER" "insert into t0_default values (1,'abc')"
+    run_presto_query "$APP_USER" "select * from t0_default"
     echo
 
-    run_preto_query "$APP_USER" "create table t0_orc (my_id bigint, my_string varchar) with (format = 'ORC')"
-    run_preto_query "$APP_USER" "create table t0_text (my_id bigint, my_string varchar) with (format = 'TEXTFILE')"
+    run_presto_query "$APP_USER" "create table t0_orc (my_id bigint, my_string varchar) with (format = 'ORC')"
+    run_presto_query "$APP_USER" "create table t0_text (my_id bigint, my_string varchar) with (format = 'TEXTFILE')"
     echo
 
-    run_preto_query "$APP_USER" "create table t1 as select * from mdl.mdl_object_mdl_txt limit 10"
-    run_preto_query "$APP_USER" "select * from t1"
+    run_presto_query "$APP_USER" "create table t1 as select * from mdl.mdl_object_mdl_txt limit 10"
+    run_presto_query "$APP_USER" "select * from t1"
     echo
 
     cleanup_tmp_objects

mdl/assembly/assembly-descriptor.xml

@@ -47,7 +47,7 @@
             <outputDirectory>${file.separator}ldap</outputDirectory>
         </file>
         <file>
-            <source>${project.build.directory}/herd-mdl-1.2.0-tests.jar</source>
+            <source>${project.build.directory}/herd-mdl-${project.version}-tests.jar</source>
             <outputDirectory>${file.separator}mdlt</outputDirectory>
         </file>
     </files>