Merged in dev/rich/add-media-manager-cdn-2021.02.25 (pull request ela…

…stic#65)

This is the initial commit for the Media Manager CloudFront CDN

* This is the inital commit for the Media Manager CloudFront CDN.

We are using terraform to create a CloudFront CDN, with each
sandbox S3 Bucket as an origin server for the CDN.  It routes
traffic to the origin server based on the URL path, so the S3
bucket will need to have the content in the /sandbox S3 folder.

The terraform resource 'aws_cloudfront_distribution' includes
a 'cf_create' var to determine if the CDN should be created, so
this must be passed as true to create the CDN.  Additionally, it
includes a 'cf_enabled' var that must be true for the CDN to
accept traffic.

In the event both of these are true, it will add a tag 'MediaCDN'
to the engageli instance, with the value of the CDN FQDN.  The
rc.local on the engageli instance has also been modified to get
this tag, and if the tag exists will modify the 'storagebaseurl'
in the media manager local.yml file, so when playing video clips
the content will be served via the CDN.  If the tag is not set,
the 'storagebaseurl' will continue to point directly to the S3
bucket as it is today.
modified:   aws/ams-cluster-v1-tf/main.tf
new file:   aws/ams-cluster-v1-tf/media-manager.tf
modified:   aws/ams-cluster-v1-tf/variables.tf
modified:   instance-files/engageli/etc/rc.local

* Added CDN DNS alias 'stack-name-cdn.domain'
modified:   aws/ams-cluster-v1-tf/main.tf
modified:   aws/ams-cluster-v1-tf/media-manager.tf
modified:   aws/ams-cluster-v1-tf/variables.tf

* Implement workaround for terraform bug causing origin servers
to be dropped and added on each apply.
modified:   aws/ams-cluster-v1-tf/media-manager.tf
modified:   aws/ams-cluster-v1-tf/variables.tf

* additional fixup and todo's
modified:   media-manager.tf

* Fixed case where CDN not created, to not create DNS for alias
modified:   aws/ams-cluster-v1-tf/main.tf
modified:   aws/ams-cluster-v1-tf/media-manager.tf

* Changes to address review feedback.
modified:   aws/ams-cluster-v1-tf/main.tf
modified:   aws/ams-cluster-v1-tf/media-manager.tf
modified:   aws/ams-cluster-v1-tf/variables.tf

* Merge branch 'master' of bitbucket.org:engageli/devops into dev/rich/add-media-manager-cdn-2021.02.25

modified:   admin-tools/package-lock.json
modified:   admin-tools/package.json
modified:   aws/ams-cluster-v1-tf/main.tf
modified:   aws/ams-cluster-v1-tf/variables.tf
modified:   packer/tag_all.sh
modified:   provisioning/provision-dequeue.sh

* Changed default for var.cf_cert_arn to production wildcard for
*.p1.engageli-prod.com
modified:   aws/ams-cluster-v1-tf/variables.tf

* fixed missing curly bracket after merge
modified:   aws/ams-cluster-v1-tf/variables.tf

Approved-by: Gideon Avida

aws/ams-cluster-v1-tf/main.tf

@@ -483,6 +483,7 @@ resource "aws_instance" "engageli" {
     Sandboxes   = join(",", var.sandboxes)
     StackName   = var.stack_name
     EnableDQ    = var.num_dequeue > 0 ? "true" : "false"
+    MediaCDN    = (var.cf_create && var.cf_enabled) ? join("", aws_route53_record.mm-cdn-alias[*].fqdn) : ""
   }
 }
 
@@ -639,11 +640,11 @@ resource "aws_lb_target_group_attachment" "engageli" {
 }
 
 resource "aws_lb_target_group" "ams" {
-  name        = "tf-${var.stack_name}-ams-external-tg"
-  port        = 5080
-  protocol    = "HTTP"
-  vpc_id      = aws_vpc.main.id
-  target_type = "instance"
+  name                          = "tf-${var.stack_name}-ams-external-tg"
+  port                          = 5080
+  protocol                      = "HTTP"
+  vpc_id                        = aws_vpc.main.id
+  target_type                   = "instance"
   load_balancing_algorithm_type = "least_outstanding_requests"
   health_check {
     enabled = true
@@ -655,11 +656,11 @@ resource "aws_lb_target_group" "ams" {
 # FIXME: currently duplicating AMS target groups so we can have both internal
 # and external load balancers
 resource "aws_lb_target_group" "ams_internal" {
-  name        = "tf-${var.stack_name}-ams-internal-tg"
-  port        = 5080
-  protocol    = "HTTP"
-  vpc_id      = aws_vpc.main.id
-  target_type = "instance"
+  name                          = "tf-${var.stack_name}-ams-internal-tg"
+  port                          = 5080
+  protocol                      = "HTTP"
+  vpc_id                        = aws_vpc.main.id
+  target_type                   = "instance"
   load_balancing_algorithm_type = "least_outstanding_requests"
   health_check {
     enabled = true
@@ -788,6 +789,12 @@ resource "aws_s3_bucket" "mm" {
     # expose_headers  = ["ETag"]
     # max_age_seconds = 3000
   }
+
+  tags = {
+    Sandbox   = each.key
+    Terraform = true
+    StackName = var.stack_name
+  }
 }
 
 data "aws_caller_identity" "my_account" {}

aws/ams-cluster-v1-tf/media-manager.tf

@@ -0,0 +1,107 @@
+# TODO: move media-manager S3 resorces from main.tf
+
+resource "aws_cloudfront_distribution" "mm_cloudfront_cdn" {
+  count           = var.cf_create ? 1 : 0
+  comment         = "Media Manager CDN for terraform stack: ${var.stack_name}"
+  enabled         = var.cf_enabled
+  price_class     = var.cf_price_class
+  is_ipv6_enabled = false # TODO: Create DNS AAAA record to support ipv6
+  aliases         = [trimsuffix("${var.stack_name}-cdn.${var.domain}", ".")]
+
+  viewer_certificate {
+    #cloudfront_default_certificate = true
+    acm_certificate_arn      = var.cf_cert_arn
+    ssl_support_method       = var.cf_ssl_support_method
+    minimum_protocol_version = var.cf_minimum_protocol_version
+  }
+
+  dynamic "origin" {
+    for_each = [for b in aws_s3_bucket.mm : {
+      name = b.bucket_domain_name
+      id   = lookup(b.tags, "Sandbox", b.bucket)
+    }]
+    content {
+      domain_name = origin.value.name
+      origin_id   = origin.value.id
+      origin_path = ""
+      # This section is a required setting, but terraform has a bug that causes it to drop and
+      # recreate each origin server on all apply if origin_access_identity null.  The workaround
+      # is to remove, reference: https://github.com/hashicorp/terraform/issues/7930
+      # If we include a value for origin_access_identity, it prevents direct access to the S3
+      # bucket, so it forces all requests to use the CDN or they will fail.
+      #   s3_origin_config {
+      #     origin_access_identity = ""
+      #   }
+    }
+  }
+
+  default_cache_behavior {
+    allowed_methods        = ["GET", "HEAD"]
+    cached_methods         = ["GET", "HEAD"]
+    viewer_protocol_policy = "https-only"
+    compress               = true
+    target_origin_id       = element(var.sandboxes, 0) #TODO: Get from S3 Sandbox tag, not used anyway
+    forwarded_values {
+      query_string = false
+      headers      = ["Origin"]
+      cookies {
+        forward = "none"
+      }
+    }
+  }
+
+  dynamic "ordered_cache_behavior" {
+    for_each = [for b in aws_s3_bucket.mm : {
+      id = lookup(b.tags, "Sandbox", b.bucket)
+    }]
+    content {
+      path_pattern           = "${ordered_cache_behavior.value.id}/*"
+      allowed_methods        = ["GET", "HEAD"]
+      cached_methods         = ["GET", "HEAD"]
+      viewer_protocol_policy = "https-only"
+      compress               = true
+      target_origin_id       = ordered_cache_behavior.value.id
+      forwarded_values {
+        query_string = false
+        headers      = ["Origin"]
+        cookies {
+          forward = "none"
+        }
+      }
+    }
+  }
+
+  restrictions {
+    geo_restriction {
+      restriction_type = "none"
+    }
+  }
+
+  tags = {
+    Name      = "tf-${var.stack_name}-mm-cloudfront-cdn"
+    Terraform = true
+    StackName = var.stack_name
+  }
+
+}
+
+output "mm_CDN_domain_name" {
+  value = join("", aws_cloudfront_distribution.mm_cloudfront_cdn[*].domain_name)
+}
+
+resource "aws_route53_record" "mm-cdn-alias" {
+  count   = var.cf_create ? 1 : 0
+  zone_id = var.r53_zone_id
+  name    = "${var.stack_name}-cdn.${var.domain}"
+  type    = "A"
+
+  alias {
+    name                   = aws_cloudfront_distribution.mm_cloudfront_cdn[0].domain_name
+    zone_id                = aws_cloudfront_distribution.mm_cloudfront_cdn[0].hosted_zone_id
+    evaluate_target_health = false
+  }
+}
+
+output "mm_CDN_FQDN" {
+  value = join("", aws_route53_record.mm-cdn-alias[*].fqdn)
+}

aws/ams-cluster-v1-tf/variables.tf

@@ -175,6 +175,47 @@ variable "redis_parameter_group_name" {
   default     = "default.redis6.x"
 }
 
+# Variables for CloudFront CDN
+# https://docs.aws.amazon.com/AmazonCloudFront/latest/DeveloperGuide/distribution-web-values-specify.html
+
+variable "cf_create" {
+  description = "Set to false to prevent the creating CloudFront resource"
+  type        = bool
+  default     = false
+}
+
+variable "cf_enabled" {
+  description = "Set to false to prevent CloudFront from accepting end user requests"
+  type        = bool
+  default     = true
+}
+
+# https://docs.aws.amazon.com/acm/latest/userguide/acm-regions.html
+# Default set to wildcard cert for prod1: *.p1.engageli-prod.com
+variable "cf_cert_arn" {
+  description = "ARN for cert, must be located in us-east-1"
+  type        = string
+  default     = "arn:aws:acm:us-east-1:503455782123:certificate/d7485579-2108-4229-a561-fe3299a8b4a7"
+}
+
+variable "cf_price_class" {
+  description = "Price Class determines what edge locations are included"
+  type        = string
+  default     = "PriceClass_100"
+}
+
+variable "cf_minimum_protocol_version" {
+  description = "The minimum version of the SSL protocol"
+  type        = string
+  default     = "TLSv1.2_2019"
+}
+
+variable "cf_ssl_support_method" {
+  description = "CloudFront to serve HTTPS requests - vip uses a dedicated IP"
+  type        = string
+  default     = "sni-only"
+}
+
 #https://docs.aws.amazon.com/elasticloadbalancing/latest/application/create-https-listener.html
 variable "aws_lb_ssl_policy" {
   description = "SSL Policy for aws_lb_listener"

instance-files/engageli/etc/rc.local

@@ -46,6 +46,7 @@ export PGUSER=postgres
 export REDIS_HOST=$(get_tag_value RedisHost)
 export REDIS_SECRET=$(get_tag_value RedisSecret)
 ENABLE_DQ=$(get_tag_value EnableDQ)
+MEDIA_CDN=$(get_tag_value MediaCDN)
 
 # Initial DB password is from AWS Secret Manager
 # FIXME: Get rid of secret once it's not needed anymore (need to signal
@@ -120,6 +121,7 @@ do
         -n $SANDBOX \
         -p $(($SANDBOX_PORT + 1000)) \
         -b $BUCKET \
+        -c "${MEDIA_CDN}" \
         -r $REGION \
         --redis-host $REDIS_HOST \
         --redis-secret $REDIS_SECRET