Develop (#96)

* metastor changes * 1.5.0 (#95) * updated to newer version * specify only two subnets for elasticsearch domain as AWS doesn't choose first two by default now. (#88) * 1.0.5 Release - Removed duplicate parameters in CFT templates - Fixed bug with ns-auth sync config applying to only Herd installations - Fixed incremental DB script installation logic - Upgraded Herd and Shepherd (Herd-UI) versions - Upgraded Herd DB engine version - Fixed issue where certain DB instance types were not available - Update herd-ui to the latest available version: 0.78.0 * Updated OSS version
FINRAOS · Nov 8, 2019 · 542237f · 542237f
1 parent 15c5314
commit 542237f
Show file tree

Hide file tree

Showing 27 changed files with 737 additions and 594 deletions.
diff --git a/mdl/src/main/cft/installMDL.yml b/mdl/src/main/cft/installMDL.yml
@@ -20,7 +20,7 @@ Parameters:
     Description: >
       MDL release version to use. [REQUIRED]
     Type: String
-    Default: '1.4.0'
+    Default: '1.5.0'
   DeployComponents:
     Type: String
     Default: All
@@ -44,11 +44,11 @@ Parameters:
       - 'false'
     Type: String
   HerdDBClass:
-    Default: db.t2.medium
+    Default: db.m4.large
     Description: Database instance class for Herd
     Type: String
   MetastorDBClass:
-    Default: db.m3.medium
+    Default: db.m4.large
     Description: Database instance class for Metastor
     Type: String
   HerdDBSize:

diff --git a/mdl/src/main/cft/mdl.yml b/mdl/src/main/cft/mdl.yml
@@ -36,11 +36,11 @@ Parameters:
     AllowedValues: [true, false]
     Type: String
   HerdVersion:
-    Default: '0.81.0'
+    Default: '0.107.0'
     Description: Herd release version to use.
     Type: String
   HerdUIVersion:
-    Default: '0.41.0'
+    Default: '0.78.0'
     Description: Herd-UI release version to use.
     Type: String
   HerdDBClass:
@@ -299,7 +299,6 @@ Resources:
         HerdSecurityGroup: !Join ['', [/app/MDL/, !Ref MDLInstanceName, /, !Ref Environment, /SecurityGroup/Herd]]
         MDLServerDeploymentRole: !Join ['', [/app/MDL/, !Ref MDLInstanceName, /, !Ref Environment, /IAM/MDLServerDeploymentRole]]
         CertificateArn: !Ref CertificateArn
-        ShepherdWebSiteBucketUrl: !Join ['', [/app/MDL/, !Ref MDLInstanceName, /, !Ref Environment, /S3/URL/Shepherd]]
         ShepherdS3BucketName: !Join ['', [/app/MDL/, !Ref MDLInstanceName, /, !Ref Environment, /S3/Shepherd]]
         HerdBucketName: !Join ['', [/app/MDL/, !Ref MDLInstanceName, /, !Ref Environment, /S3/Herd]]
         DeploymentBucketName: !Ref DeploymentBucketName
@@ -321,6 +320,7 @@ Resources:
         HostedZoneName: !Ref HostedZoneName
         MDLInstanceName: !Ref MDLInstanceName
         Environment: !Ref Environment
+        DeployComponents: !Ref DeployComponents
         VpcIdParameterKey: !Join ['/', ['/global', !Ref MDLInstanceName, !Ref Environment, 'VPC/ID']]
         PrivateSubnetsParameterKey: !Join ['/', ['/global', !Ref MDLInstanceName, !Ref Environment, 'VPC/SubnetIDs/private']]
         PublicSubnetsParameterKey: !Join ['/', ['/global', !Ref MDLInstanceName, !Ref Environment, 'VPC/SubnetIDs/public']]

diff --git a/mdl/src/main/cft/mdlBdsql.yml b/mdl/src/main/cft/mdlBdsql.yml
@@ -233,7 +233,6 @@ Resources:
             hive.stats.autogather: 'false'
             javax.jdo.option.ConnectionUserName: 'MS_Hive_0_13'
             javax.jdo.option.ConnectionPassword: '{{HIVE_PASSWORD}}'
-            hive.metastore.schema.verification: false
             javax.jdo.option.ConnectionURL: !Sub 'jdbc:mysql://${MetastorDBHostName}:3306/metastor?trustServerCertificate=true&createDatabaseIfNotExist=false&useSSL=true&requireSSL=true'
             hive.metastore.warehouse.dir: !Join
               - ''

diff --git a/mdl/src/main/cft/mdlCreateNsAuthSyncUtil.yml b/mdl/src/main/cft/mdlCreateNsAuthSyncUtil.yml
@@ -97,6 +97,16 @@ Resources:
       ManagedPolicyArns:
         - 'arn:aws:iam::aws:policy/AmazonSSMReadOnlyAccess'
         - 'arn:aws:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole'
+        - 'arn:aws:iam::aws:policy/service-role/AWSLambdaENIManagementAccess'
+      Policies:
+        - PolicyName: root
+          PolicyDocument:
+            Version: 2012-10-17
+            Statement:
+              - Action:
+                  - 'ec2:DetachNetworkInterface'
+                Effect: Allow
+                Resource: '*'
   NsAuthUtilityArnParameter:
     Type: AWS::SSM::Parameter
     Properties:
@@ -127,65 +137,6 @@ Resources:
       Principal: sns.amazonaws.com
       SourceArn: !Ref NsAuthChangeNotificationSnsTopic
       FunctionName: !GetAtt 'NsAuthSyncLambdaFunction.Arn'
-  ManageEniLifecycleFunction:
-      Type: AWS::Lambda::Function
-      DependsOn: NsAuthSyncLambdaFunction
-      Properties:
-        Handler: index.handler
-        Role: !GetAtt LambdaExecutionRole.Arn
-        Code:
-          ZipFile: !Sub |
-            var response = require('cfn-response');
-            var AWS = require('aws-sdk');
-
-            exports.handler = function(event, context) {
-              if (event.RequestType != 'Delete') {
-                response.send(event, context, response.SUCCESS, {});
-                return;
-              }
-              var ec2 = new AWS.EC2();
-              var params = {
-                Filters: [
-                  {
-                    Name: 'group-id',
-                    Values: event.ResourceProperties.SecurityGroups
-                  },
-                  {
-                    Name: 'description',
-                    Values: ['AWS Lambda VPC ENI: *']
-                  }
-                ]
-              };
-
-              console.log("Deleting attachments!");
-              // Detach, then delete ENIs which were spinned up by AWS for our VPC Lambda
-              ec2.describeNetworkInterfaces(params).promise().then(function(data) {
-                console.log("Got Interfaces:\n", JSON.stringify(data));
-                return Promise.all(data.NetworkInterfaces.map(function(networkInterface) {
-                  var networkInterfaceId = networkInterface.NetworkInterfaceId;
-                  var attachmentId = networkInterface.Attachment.AttachmentId;
-                  return ec2.detachNetworkInterface({AttachmentId: attachmentId}).promise().then(function(data) {
-                    return ec2.waitFor('networkInterfaceAvailable', {NetworkInterfaceIds: [networkInterfaceId]}).promise();
-                  }).then(function(data) {
-                    console.log("Detached Interface, deleting:\n", networkInterfaceId);
-                    return ec2.deleteNetworkInterface({NetworkInterfaceId: networkInterfaceId}).promise();
-                  });
-                }));
-              }).then(function(data) {
-                console.log("Success!");
-                response.send(event, context, response.SUCCESS, {});
-              }).catch(function(err) {
-                console.log("Failure:\n", JSON.stringify(err));
-                response.send(event, context, response.FAILED, {});
-              });
-            };
-        Timeout: 300
-        Runtime: nodejs4.3
-  VPCDestroyENI:
-    Type: Custom::VPCDestroyENI
-    Properties:
-      ServiceToken: !GetAtt 'ManageEniLifecycleFunction.Arn'
-      SecurityGroups: [!Ref NsAuthSyncUtilitySecurityGroupParameter]
 Outputs:
   LambdaFunctionName:
     Description: Function name of the ns-auth sync utility lambda.

diff --git a/mdl/src/main/cft/mdlCreateS3Buckets.yml b/mdl/src/main/cft/mdlCreateS3Buckets.yml
@@ -208,22 +208,6 @@ Resources:
       Type: String
       Value: !If [EnableSSLAndAuth, !Ref CloudfrontS3Bucket, !Ref S3BucketForStaticPages]
       Description: Name of Shepherd S3 bucket
-  ShepherdS3BucketDomainParameter:
-    Type: 'AWS::SSM::Parameter'
-    Condition: EnableSSLAndAuth
-    Properties:
-      Name: !Join
-        - ''
-        - - /app/MDL/
-          - !Ref MDLInstanceName
-          - /
-          - !Ref Environment
-          - /S3/Domain/Shepherd
-      Type: String
-      Value: !GetAtt
-        - CloudfrontS3Bucket
-        - DomainName
-      Description: Domain name of Shepherd S3 bucket
   ShepherdS3BucketURLParameter:
     Type: 'AWS::SSM::Parameter'
     Condition: EnableSSLAndAuth

diff --git a/mdl/src/main/cft/mdlHerd.yml b/mdl/src/main/cft/mdlHerd.yml
@@ -34,9 +34,9 @@ Parameters:
   CreateSQS:
     Description: Create SQS - true || false
     Type: AWS::SSM::Parameter::Value<String>
-  ShepherdWebSiteBucketUrl:
-    Description: Shepherd website URL
-    Type: AWS::SSM::Parameter::Value<String>
+  DeployComponents:
+    Description: Components requested for deployment
+    Type: String
   ShepherdS3BucketName:
     Description: 'The bucket name of Shepherd '
     Type: AWS::SSM::Parameter::Value<String>
@@ -484,6 +484,10 @@ Resources:
 
                     environment=
                   - !Ref Environment
+                  - |-
+
+                    deployComponents=
+                  - !Ref DeployComponents
                   - |-
 
                     herdVersion=

diff --git a/mdl/src/main/cft/mdlHerdRds.yml b/mdl/src/main/cft/mdlHerdRds.yml
@@ -26,15 +26,14 @@ Parameters:
     AllowedPattern: '[a-zA-Z_][a-zA-Z0-9_]*'
     ConstraintDescription: must begin with a letter and contain alphanumeric characters and _.
   HerdDBClass:
-    Default: db.m4.large
     Description: Database instance class
     Type: String
   HerdDBEngine:
     Default: postgres
     Description: Postgres RDS database Engine
     Type: String
   HerdDBEngineVersion:
-    Default: 9.5.4
+    Default: 9.5.15
     Description: Postgres database version
     Type: String
   HerdDBSize:

diff --git a/mdl/src/main/cft/mdlMetastorRds.yml b/mdl/src/main/cft/mdlMetastorRds.yml
@@ -29,7 +29,6 @@ Parameters:
     Description: VPC Parameter key name in system store
     Type: String
   MetastorDBClass:
-    Default: db.m4.large
     Description: Database instance class
     Type: String
   MetastorDBEngine:

diff --git a/mdl/src/main/cft/mdlShepherd.yml b/mdl/src/main/cft/mdlShepherd.yml
@@ -28,11 +28,6 @@ Parameters:
   CertificateArn:
     Description: Certificate Arn for MDL
     Type: String
-  EnableSSLAndAuth:
-    Default: 'false'
-    Description: Whether to enable SSL and Auth
-    ConstraintDescription: Must specify true or false
-    Type: String
   EnableSSLAndAuth:
     Default: 'true'
     Description: Whether to enable Authentication/SSL

diff --git a/mdl/src/main/herd/scripts/installDBForHerd.sh b/mdl/src/main/herd/scripts/installDBForHerd.sh
@@ -200,22 +200,14 @@ if [ "${herdRollingDeployment}" = "true" ] ; then
   echo "Requested version: ${newVersion}"
 fi
 
-while [ ${initialVersion} -lt ${newVersion} ]
+boundary=`expr ${newVersion} - 1`
+
+for i in $(seq ${initialVersion} ${boundary})
 do
-    if [ ${initialVersion} -lt 10 ]; then
-        next=$((10#${initialVersion}+1))
-        pre="0${initialVersion}"
-        if [ ${initialVersion} -eq 9 ]; then
-            post="${next}"
-        else
-            post="0${next}"
-        fi
-    else
-        pre="${initialVersion}"
-        next=$((10#${initialVersion}+1))
-        post="${next}"
-    fi
-    ((initialVersion++))
+
+  printf -v pre "%03d" $i
+  var=`expr $i + 1`
+  printf -v post "%03d" $var
 
 # Apply incremental upgrade scripts to the Herd DB
 echo "Running incremental DB script: ${deployLocation}/sql/herd.postgres.0.${pre}.0-to-0.${post}.0.upgrade.sql"
@@ -261,13 +253,14 @@ if [ "${herdRollingDeployment}" = "false" ] ; then
         # Add namespace authorization admin permissions for the app-admin user
         execute_cmd "psql --set ON_ERROR_STOP=on --host ${herdDatabaseHost} --port 5432 -c \"INSERT INTO dmrowner.user_tbl VALUES ('${admin_user_url}', 'USER', 'ADMIN', current_timestamp, current_timestamp, 'SYSTEM', 'SYSTEM', '${PGUSER}', 'Y', 'Y');\""
 
-
         # Add Herd namespace-auth SNS topic configuration
-        sns_arn=$(aws ssm get-parameter --name /app/MDL/${mdlInstanceName}/${environment}/Resources/SNS/UserNamespaceChgTopicArn --region ${region} --output text --query Parameter.Value)
-        echo "Inserting SNS config for user-namespace authorization status changes. SNS Arn: ${sns_arn}"
-        execute_cmd "sed -i \"s/{{SNS_TOPIC_ARN}}/${sns_arn}/g\" ${deployLocation}/sql/nsAuthSnsConfig.sql"
-        execute_cmd "sed -i \"s/{{ENVIRONMENT}}/${environment}/g\" ${deployLocation}/sql/nsAuthSnsConfig.sql"
-        execute_cmd "psql --set ON_ERROR_STOP=on --host ${herdDatabaseHost} --port 5432 -f ${deployLocation}/sql/nsAuthSnsConfig.sql"
+        if [ ${deployComponents} =  "All" ] || [ ${deployComponents} == "BDSQL" ] ; then
+          sns_arn=$(aws ssm get-parameter --name /app/MDL/${mdlInstanceName}/${environment}/Resources/SNS/UserNamespaceChgTopicArn --region ${region} --output text --query Parameter.Value)
+          echo "Inserting SNS config for user-namespace authorization status changes. SNS Arn: ${sns_arn}"
+          execute_cmd "sed -i \"s/{{SNS_TOPIC_ARN}}/${sns_arn}/g\" ${deployLocation}/sql/nsAuthSnsConfig.sql"
+          execute_cmd "sed -i \"s/{{ENVIRONMENT}}/${environment}/g\" ${deployLocation}/sql/nsAuthSnsConfig.sql"
+          execute_cmd "psql --set ON_ERROR_STOP=on --host ${herdDatabaseHost} --port 5432 -f ${deployLocation}/sql/nsAuthSnsConfig.sql"
+        fi
 
         # ensure that jms publishing is enabled
         execute_cmd "psql --set ON_ERROR_STOP=on --host ${herdDatabaseHost} --port 5432 -c \"DELETE FROM cnfgn WHERE cnfgn_key_nm = 'jms.listener.enabled';\""

diff --git a/metastor/managedObjectLoader/pom.xml b/metastor/managedObjectLoader/pom.xml
@@ -19,7 +19,7 @@
 	<parent>
 		<artifactId>metastore</artifactId>
 		<groupId>org.finra.herd-mdl.metastore</groupId>
-		<version>1.2.25</version>
+		<version>1.2.38</version>
 	</parent>
 
 	<modelVersion>4.0.0</modelVersion>