PXP-7805 Audit Service SQS

doc/awsrole.md

@@ -42,13 +42,15 @@ Options:
 
 ### attach-policy
 
-Attaches a policy to a role
+Attaches a policy to a user or role
 ```
-  gen3 awsrole attach-policy <rolename> <policyARN>
+  gen3 awsrole attach-policy <policyARN> --role-name <rolename>
+  gen3 awsrole attach-policy <policyARN> --user-name <username>
 ```
 Options:
-  - rolename: name of role to attach policy to
   - policyARN: arn of policy to attach to role
+  - rolename/username: name of entity to attach policy to
+  - --force-aws-cli: use the AWS CLI even when a Terraform module exists
 
 
 ### sa-ar-policy $serviceAccountName

doc/sqs.md

@@ -0,0 +1,23 @@
+# TL;DR
+
+Create and interact with AWS SQS queues.
+
+## Use
+
+### info
+
+Returns the SQS URL for the provided SQS.
+```
+   gen3 sqs info <sqsName>
+```
+Options:
+  - sqsName: name of SQS to fetch the URL for.
+
+### create-queue
+
+Creates a new SQS queue, along with 2 policies to push and pull from the queue. Returns an SQS URL and the policies ARNs.
+```
+  gen3 s3 create-queue <sqsName>
+```
+Options:
+  - sqsName: name of SQS to create.

gen3/bin/awsrole.sh

@@ -220,43 +220,94 @@ gen3_awsrole_info() {
 }
 
 #
-# Attach policy to a role
-# 
-# @param rolename
-# @param policyarn
+# Attach a policy to a user or role
+#
+# @param policyArn
+# @param entityTypeFlag: "--user-name" or "--role-name"
+# @param entityName
+# @param forceAwsCli: "--force-aws-cli" to use the AWS CLI even when a Terraform module exists
 #
 gen3_awsrole_attachpolicy() {
-  local rolename=$1
-  local policyarn=$2
-
-  # verify policy and role exist
-  if ! gen3_aws_run aws iam get-role --role-name $rolename > /dev/null 2>&1; then
-    gen3_log_err "Unable to find role with given name"
+  local policyArn=$1
+  local entityTypeFlag=$2
+  local entityName=$3
+  local forceAwsCli=$4
+
+  if [[ -z "$entityName" ]]; then
+    gen3_log_err "User/Role name must not be empty"
     return 1
   fi
-  if ! gen3_aws_run aws iam get-policy --policy-arn $policyarn > /dev/null 2>&1; then
-    gen3_log_err "Unable to find policy with given arn"
+
+  # check the iam entity type
+  local entityType
+  if [[ $entityTypeFlag == "--user-name" ]]; then
+    entityType="user"
+  elif [[ $entityTypeFlag == "--role-name" ]]; then
+    entityType="role"
+  else
+    gen3_log_err "Invalid entity type provided: $entityTypeFlag"
     return 1
   fi
 
-  # attach using terraform
-  gen3 workon default ${rolename}_role_policy_attachment
-  gen3 cd
-  gen3_log_info "In terraform workspace ${GEN3_WORKSPACE}"
-  cat << EOF > config.tfvars
-role="$rolename"
-policy_arn="$policyarn"
-EOF
-  if ! gen3 tfplan 2>&1; then
+  # verify policy exists
+  if ! gen3_aws_run aws iam get-policy --policy-arn $policyArn > /dev/null 2>&1; then
+    gen3_log_err "Unable to find policy with given arn"
     return 1
   fi
 
-  if ! gen3 tfapply 2>&1; then
-    gen3_log_err "Unexpected error running gen3 tfapply. Please cleanup workspace in ${GEN3_WORKSPACE}"
+  local alreadyHasPolicy=$(_entity_has_policy $entityType $entityName $policyArn)
+  if [[ $? != 0 ]]; then
+    gen3_log_err "Failed to determine if entity already has policy"
     return 1
   fi
+  if [[ "true" == "$alreadyHasPolicy" ]]; then
+    gen3_log_info "Policy already attached"
+    return 0
+  fi
 
-  gen3 trash --apply
+  # attach the policy to the user (AWS CLI), or to the role if forcing AWS CLI
+  if [[ $entityTypeFlag == "--user-name" || $forceAwsCli == "--force-aws-cli" ]]; then
+    local attachStdout
+    attachStdout=$(gen3_aws_run aws iam attach-${entityType}-policy --${entityType}-name $entityName --policy-arn $policyArn 2>&1)
+    if [[ $? != 0 ]]; then
+      local errMsg=$(
+        cat << EOF
+Failed to attach policy:
+$attachStdout
+EOF
+      )
+      gen3_log_err $errMsg
+      return 1
+    fi
+    gen3_log_info "Successfully attached policy"
+
+  # attach the policy to the role (terraform)
+  elif [[ $entityTypeFlag == "--role-name" ]]; then
+    # verify role exists
+    if ! gen3_aws_run aws iam get-role --role-name $entityName > /dev/null 2>&1; then
+      gen3_log_err "Unable to find role with given name"
+      return 1
+    fi
+
+    # attach policy
+    gen3 workon default ${entityName}_role_policy_attachment
+    gen3 cd
+    gen3_log_info "In terraform workspace ${GEN3_WORKSPACE}"
+    cat << EOF > config.tfvars
+role="$entityName"
+policy_arn="$policyArn"
+EOF
+    if ! gen3 tfplan 2>&1; then
+      return 1
+    fi
+
+    if ! gen3 tfapply 2>&1; then
+      gen3_log_err "Unexpected error running gen3 tfapply. Please cleanup workspace in ${GEN3_WORKSPACE}"
+      return 1
+    fi
+
+    gen3 trash --apply
+  fi
 }
 
 #

gen3/bin/kube-setup-access-backend.sh

@@ -73,7 +73,7 @@ setup_access_backend() {
 
     local saName=$(echo "access-${hostname//./-}" | head -c63)
     if ! g3kubectl get sa "$saName" > /dev/null 2>&1; then
-      local role_name
+      local roleName
       if ! g3kubectl get sa access-backend-sa > /dev/null 2>&1; then
         roleName="$(gen3 api safe-name access-backend)"
         gen3 awsrole create "$roleName" access-backend-sa
@@ -103,10 +103,10 @@ EOM
         aws iam create-policy --policy-name $roleName --policy-document "$policy"
         accountNumber=$(aws sts get-caller-identity | jq -r .Account)
         sleep 15
-        gen3 awsrole attach-policy $roleName arn:aws:iam::$accountNumber:policy/$roleName
+        gen3 awsrole attach-policy arn:aws:iam::$accountNumber:policy/$roleName --role-name $roleName
       fi
       gen3_log_info "created service account '${saName}' with dynamodb access"
-      gen3_log_info "created role name '${role_name}'"
+      gen3_log_info "created role name '${roleName}'"
       # TODO do I need the following: ???
       # gen3 s3 attach-bucket-policy "$bucketName" --read-write --role-name "${role_name}"
       # gen3_log_info "attached read-write bucket policy to '${bucketName}' for role '${role_name}'"

gen3/bin/kube-setup-audit-service.sh

@@ -7,8 +7,8 @@ source "${GEN3_HOME}/gen3/lib/utils.sh"
 gen3_load "gen3/gen3setup"
 
 
-setup_database() {
-  gen3_log_info "setting up audit-service..."
+setup_database_and_config() {
+  gen3_log_info "setting up audit-service DB and config"
 
   if g3kubectl describe secret audit-g3auto > /dev/null 2>&1; then
     gen3_log_info "audit-g3auto secret already configured"
@@ -18,6 +18,7 @@ setup_database() {
     gen3_log_err "skipping db setup in non-adminvm environment"
     return 0
   fi
+
   # Setup config file that audit-service consumes
   if [[ ! -f "$secretsFolder/audit-service-config.yaml" || ! -f "$secretsFolder/base64Authz.txt" ]]; then
     local secretsFolder="$(gen3_secrets_folder)/g3auto/audit"
@@ -31,6 +32,9 @@ setup_database() {
       gen3_log_err "dbcreds not present in Gen3Secrets/"
       return 1
     fi
+
+    availability_zone=$(curl http://169.254.169.254/latest/meta-data/placement/availability-zone -s)
+    region=$(echo ${availability_zone::-1})
 
     cat - > "$secretsFolder/audit-service-config.yaml" <<EOM
 ####################
@@ -40,6 +44,13 @@ setup_database() {
 # whether to enable debug logs
 DEBUG: true
 
+PULL_FROM_QUEUE: true
+QUEUE_CONFIG:
+  type: aws_sqs
+  aws_sqs_config:
+    sqs_url: ${sqsUrl}
+    region: ${region}
+
 ####################
 # DATABASE         #
 ####################
@@ -55,13 +66,36 @@ EOM
   gen3 secrets sync 'setup audit-g3auto secrets'
 }
 
+setup_audit_sqs() {
+  local sqsName="$(gen3 api safe-name audit-sqs)"
+  sqsInfo="$(gen3 sqs create-queue-if-not-exist $sqsName)" || exit 1
+  sqsUrl="$(jq -e -r '.["url"]' <<< "$sqsInfo")" || { echo "Cannot get 'sqs-url' from output: $sqsInfo"; exit 1; }
+  sqsArn="$(jq -e -r '.["arn"]' <<< "$sqsInfo")" || { echo "Cannot get 'sqs-arn' from output: $sqsInfo"; exit 1; }
+
+  # audit-service can pull messages from the audit queue
+  local saName="audit-service-sa"
+  local roleName="$(gen3 api safe-name audit-sqs-receiver)" || exit 1
+  gen3_log_info "setting up service account '$saName' with role '${roleName}'"
+  if ! gen3 awsrole info "$roleName" > /dev/null; then # create role
+    gen3 awsrole create "$roleName" "$saName" || exit 1
+  fi
+  gen3 sqs attach-receiver-policy-to-role $sqsArn $roleName || exit 1
+}
+
+gen3_log_info "setting up audit-service..."
+
 if ! g3k_manifest_lookup '.versions["audit-service"]' 2> /dev/null; then
   gen3_log_info "kube-setup-audit-service exiting - audit-service not in manifest"
   exit 0
 fi
 
-if ! setup_database; then
-  gen3_log_err "kube-setup-audit-service bailing out - database failed setup"
+if ! setup_audit_sqs; then
+  gen3_log_err "kube-setup-audit-service bailing out - failed to setup audit SQS"
+  exit 1
+fi
+
+if ! setup_database_and_config; then
+  gen3_log_err "kube-setup-audit-service bailing out - database/config failed setup"
   exit 1
 fi
 

gen3/bin/kube-setup-fence.sh

@@ -8,6 +8,22 @@
 source "${GEN3_HOME}/gen3/lib/utils.sh"
 gen3_load "gen3/lib/kube-setup-init"
 
+setup_audit_sqs() {
+  local sqsName="$(gen3 api safe-name audit-sqs)"
+  sqsInfo="$(gen3 sqs create-queue-if-not-exist $sqsName)" || exit 1
+  sqsUrl="$(jq -e -r '.["url"]' <<< "$sqsInfo")" || { echo "Cannot get 'sqs-url' from output: $sqsInfo"; exit 1; }
+  sqsArn="$(jq -e -r '.["arn"]' <<< "$sqsInfo")" || { echo "Cannot get 'sqs-arn' from output: $sqsInfo"; exit 1; }
+
+  # fence can push messages to the audit queue
+  local saName="fence-sa"
+  local roleName="$(gen3 api safe-name audit-sqs-sender)" || exit 1
+  gen3_log_info "setting up service account '$saName' with role '${roleName}'"
+  if ! gen3 awsrole info "$roleName" > /dev/null; then # create role
+    gen3 awsrole create "$roleName" "$saName" || exit 1
+  fi
+  gen3 sqs attach-sender-policy-to-role $sqsArn $roleName || exit 1
+}
+
 gen3 update_config fence-yaml-merge "${GEN3_HOME}/apis_configs/yaml_merge.py"
 [[ -z "$GEN3_ROLL_ALL" ]] && gen3 kube-setup-secrets
 
@@ -27,6 +43,11 @@ if [[ -f "$(gen3_secrets_folder)/creds.json" && -z "$JENKINS_HOME" ]]; then # cr
   touch "$(gen3_secrets_folder)/.rendered_fence_db"
 fi
 
+if ! setup_audit_sqs; then
+  gen3_log_err "kube-setup-fence bailing out - failed to setup audit SQS"
+  exit 1
+fi
+
 # run db migration job - disable, because this still causes locking in dcf 
 if false; then
   gen3_log_info "Launching db migrate job"

gen3/bin/s3.sh

@@ -220,34 +220,6 @@ _fetch_bucket_policy_arn() {
   fi
 }
 
-#
-# Util for checking if an entity already has a policy attached to them
-#
-# @param entityType: aws entity type (e.g. user, role...)
-# @param entityName
-# @param policyArn
-#
-_entity_has_policy() {
-  # returns true if entity already has policy, false otherwise
-  local entityType=$1
-  local entityName=$2
-  local policyArn=$3
-  # fetch policies attached to entity and check if bucket policy is already attached
-  local currentAttachedPolicies
-  currentAttachedPolicies=$(gen3_aws_run aws iam list-attached-${entityType}-policies --${entityType}-name $entityName 2>&1)
-  if [[ $? != 0 ]]; then
-    return 1
-  fi
-
-  if [[ ! -z $(echo $currentAttachedPolicies | jq '.AttachedPolicies[] | select(.PolicyArn == "'"${policyArn}"'")') ]]; then
-    echo "true"
-    return 0
-  fi
-
-  echo "false"
-  return 0
-}
-
 #
 # Attaches a bucket's read/write policy to a role
 #