feat: add support for SQS FIFO Queue

internal/pkg/cli/deploy/svc.go

@@ -1176,12 +1176,12 @@ func (d *workerSvcDeployer) stackConfiguration(in *StackRuntimeConfiguration) (*
 		return nil, err
 	}
 	conf, err := stack.NewWorkerService(stack.WorkerServiceConfig{
-		App:                 d.app,
-		Env:                 d.env.Name,
-		Manifest:            d.wsMft,
-		RawManifest:         d.rawMft,
-		RuntimeConfig:       *rc,
-		Addons:              d.addons,
+		App:           d.app,
+		Env:           d.env.Name,
+		Manifest:      d.wsMft,
+		RawManifest:   d.rawMft,
+		RuntimeConfig: *rc,
+		Addons:        d.addons,
 	})
 	if err != nil {
 		return nil, fmt.Errorf("create stack configuration: %w", err)
@@ -1207,12 +1207,12 @@ func (d *jobDeployer) stackConfiguration(in *StackRuntimeConfiguration) (*jobSta
 		return nil, err
 	}
 	conf, err := stack.NewScheduledJob(stack.ScheduledJobConfig{
-		App:                 d.app,
-		Env:                 d.env.Name,
-		Manifest:            d.jobMft,
-		RawManifest:         d.rawMft,
-		RuntimeConfig:       *rc,
-		Addons:              d.addons,
+		App:           d.app,
+		Env:           d.env.Name,
+		Manifest:      d.jobMft,
+		RawManifest:   d.rawMft,
+		RuntimeConfig: *rc,
+		Addons:        d.addons,
 	})
 	if err != nil {
 		return nil, fmt.Errorf("create stack configuration: %w", err)

internal/pkg/deploy/cloudformation/stack/testdata/workloads/worker-manifest.yml

@@ -49,6 +49,20 @@ subscribe:
       service: dogsvc
       queue:
         timeout: 1s
+    - name: mytopic
+      service: mytopic
+      queue:
+        fifo: true
+    - name: yourtopic
+      service: yourtopic
+      queue:
+        fifo:
+          content_based_deduplication: true
+          high_throughput: true
+    - name: nonfifotopic
+      service: nonfifotopic
+      queue:
+        fifo: false
 
 # Optional fields for more advanced use-cases.
 #

internal/pkg/deploy/cloudformation/stack/testdata/workloads/worker-test.stack.yml

@@ -55,6 +55,20 @@ Metadata:
           service: dogsvc
           queue:
             timeout: 1s
+        - name: mytopic
+          service: mytopic
+          queue:
+            fifo: true
+        - name: yourtopic
+          service: yourtopic
+          queue:
+            fifo:
+              content_based_deduplication: true
+              high_throughput: true
+        - name: nonfifotopic
+          service: nonfifotopic
+          queue:
+            fifo: false
 
     # Optional fields for more advanced use-cases.
     #
@@ -142,8 +156,11 @@ Resources:
               Value: !Ref EventsQueue
             - Name: COPILOT_TOPIC_QUEUE_URIS
               Value: !Sub
-                - '{"dogsvcGiveshuskiesEventsQueue":"${dogsvcgiveshuskiesURL}"}'
+                - '{"dogsvcGiveshuskiesEventsQueue":"${dogsvcgiveshuskiesURL}","mytopicMytopicfifoEventsQueue":"${mytopicmytopicfifoURL}","nonfifotopicNonfifotopicEventsQueue":"${nonfifotopicnonfifotopicURL}","yourtopicYourtopicfifoEventsQueue":"${yourtopicyourtopicfifoURL}"}'
                 - dogsvcgiveshuskiesURL: !Ref dogsvcgiveshuskiesEventsQueue
+                  mytopicmytopicfifoURL: !Ref mytopicmytopicfifoEventsQueue
+                  yourtopicyourtopicfifoURL: !Ref yourtopicyourtopicfifoEventsQueue
+                  nonfifotopicnonfifotopicURL: !Ref nonfifotopicnonfifotopicEventsQueue
           EnvironmentFiles:
           - !If
             - HasEnvFile
@@ -399,6 +416,9 @@ Resources:
               - ','
               - - !GetAtt EventsQueue.QueueName
                 - !GetAtt dogsvcgiveshuskiesEventsQueue.QueueName
+                - !GetAtt mytopicmytopicfifoEventsQueue.QueueName
+                - !GetAtt yourtopicyourtopicfifoEventsQueue.QueueName
+                - !GetAtt nonfifotopicnonfifotopicEventsQueue.QueueName
   BacklogPerTaskCalculatorRole:
     Metadata:
       'aws:copilot:description': 'An IAM role for BacklogPerTaskCalculatorFunction'
@@ -439,6 +459,9 @@ Resources:
                 Resource:
                   - !GetAtt EventsQueue.Arn
                   - !GetAtt dogsvcgiveshuskiesEventsQueue.Arn
+                  - !GetAtt mytopicmytopicfifoEventsQueue.Arn
+                  - !GetAtt yourtopicyourtopicfifoEventsQueue.Arn
+                  - !GetAtt nonfifotopicnonfifotopicEventsQueue.Arn
       ManagedPolicyArns:
         - !Sub arn:${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole
   BacklogPerTaskScheduledRule:
@@ -500,6 +523,66 @@ Resources:
               Value: !GetAtt dogsvcgiveshuskiesEventsQueue.QueueName
           Unit: Count
         TargetValue: 900
+  AutoScalingPolicymytopicmytopicfifoEventsQueue:
+    Metadata:
+      'aws:copilot:description': "An autoscaling policy to maintain 900 messages/task for mytopicmytopicfifoEventsQueue"
+    Type: AWS::ApplicationAutoScaling::ScalingPolicy
+    Properties:
+      PolicyName: !Join [ '-', [ !Ref WorkloadName, BacklogPerTask, !GetAtt mytopicmytopicfifoEventsQueue.QueueName ] ]
+      PolicyType: TargetTrackingScaling
+      ScalingTargetId: !Ref AutoScalingTarget
+      TargetTrackingScalingPolicyConfiguration:
+        ScaleInCooldown: 120
+        ScaleOutCooldown: 60
+        CustomizedMetricSpecification:
+          Namespace: !Sub '${AppName}-${EnvName}-${WorkloadName}'
+          MetricName: BacklogPerTask
+          Statistic: Average
+          Dimensions:
+            - Name: QueueName
+              Value: !GetAtt mytopicmytopicfifoEventsQueue.QueueName
+          Unit: Count
+        TargetValue: 900
+  AutoScalingPolicyyourtopicyourtopicfifoEventsQueue:
+    Metadata:
+      'aws:copilot:description': "An autoscaling policy to maintain 900 messages/task for yourtopicyourtopicfifoEventsQueue"
+    Type: AWS::ApplicationAutoScaling::ScalingPolicy
+    Properties:
+      PolicyName: !Join [ '-', [ !Ref WorkloadName, BacklogPerTask, !GetAtt yourtopicyourtopicfifoEventsQueue.QueueName ] ]
+      PolicyType: TargetTrackingScaling
+      ScalingTargetId: !Ref AutoScalingTarget
+      TargetTrackingScalingPolicyConfiguration:
+        ScaleInCooldown: 120
+        ScaleOutCooldown: 60
+        CustomizedMetricSpecification:
+          Namespace: !Sub '${AppName}-${EnvName}-${WorkloadName}'
+          MetricName: BacklogPerTask
+          Statistic: Average
+          Dimensions:
+            - Name: QueueName
+              Value: !GetAtt yourtopicyourtopicfifoEventsQueue.QueueName
+          Unit: Count
+        TargetValue: 900
+  AutoScalingPolicynonfifotopicnonfifotopicEventsQueue:
+    Metadata:
+      'aws:copilot:description': "An autoscaling policy to maintain 900 messages/task for nonfifotopicnonfifotopicEventsQueue"
+    Type: AWS::ApplicationAutoScaling::ScalingPolicy
+    Properties:
+      PolicyName: !Join [ '-', [ !Ref WorkloadName, BacklogPerTask, !GetAtt nonfifotopicnonfifotopicEventsQueue.QueueName ] ]
+      PolicyType: TargetTrackingScaling
+      ScalingTargetId: !Ref AutoScalingTarget
+      TargetTrackingScalingPolicyConfiguration:
+        ScaleInCooldown: 120
+        ScaleOutCooldown: 60
+        CustomizedMetricSpecification:
+          Namespace: !Sub '${AppName}-${EnvName}-${WorkloadName}'
+          MetricName: BacklogPerTask
+          Statistic: Average
+          Dimensions:
+            - Name: QueueName
+              Value: !GetAtt nonfifotopicnonfifotopicEventsQueue.QueueName
+          Unit: Count
+        TargetValue: 900
   Service:
     DependsOn:
       - EnvControllerAction
@@ -699,6 +782,125 @@ Resources:
             Condition:
               ArnEquals:
                 aws:SourceArn: !Join ['', [!Sub 'arn:${AWS::Partition}:sns:${AWS::Region}:${AWS::AccountId}:', !Ref AppName, '-', !Ref EnvName, '-dogsvc-giveshuskies']]
+  mytopicmytopicfifoSNSTopicSubscription:
+    Metadata:
+      'aws:copilot:description': 'A SNS subscription to topic mytopic.fifo from service mytopic'
+    Type: AWS::SNS::Subscription
+    Properties:
+      TopicArn: !Join [ '', [ !Sub 'arn:${AWS::Partition}:sns:${AWS::Region}:${AWS::AccountId}:', !Ref AppName, '-', !Ref EnvName, '-mytopic-mytopic.fifo' ] ]
+      Protocol: 'sqs'
+      Endpoint: !GetAtt mytopicmytopicfifoEventsQueue.Arn
+  mytopicmytopicfifoEventsQueue:
+    Metadata:
+      'aws:copilot:description': 'A SQS FIFO queue to buffer messages from the topic mytopic.fifo'
+    Type: AWS::SQS::Queue
+    Properties:
+      KmsMasterKeyId: !Ref EventsKMSKey
+      FifoQueue: true
+  mytopicmytopicfifoQueuePolicy:
+    Type: AWS::SQS::QueuePolicy
+    Properties:
+      Queues: [ !Ref 'mytopicmytopicfifoEventsQueue' ]
+      PolicyDocument:
+        Version: '2012-10-17'
+        Statement:
+          - Effect: Allow
+            Principal:
+              AWS:
+                - !GetAtt TaskRole.Arn
+            Action:
+              - sqs:ReceiveMessage
+              - sqs:DeleteMessage
+            Resource: !GetAtt mytopicmytopicfifoEventsQueue.Arn
+          - Effect: Allow
+            Principal:
+              Service: sns.amazonaws.com
+            Action:
+              - sqs:SendMessage
+            Resource: !GetAtt mytopicmytopicfifoEventsQueue.Arn
+            Condition:
+              ArnEquals:
+                aws:SourceArn: !Join [ '', [ !Sub 'arn:${AWS::Partition}:sns:${AWS::Region}:${AWS::AccountId}:', !Ref AppName, '-', !Ref EnvName, '-mytopic-mytopic.fifo' ] ]
+  yourtopicyourtopicfifoSNSTopicSubscription:
+    Metadata:
+      'aws:copilot:description': 'A SNS subscription to topic yourtopic.fifo from service yourtopic'
+    Type: AWS::SNS::Subscription
+    Properties:
+      TopicArn: !Join [ '', [ !Sub 'arn:${AWS::Partition}:sns:${AWS::Region}:${AWS::AccountId}:', !Ref AppName, '-', !Ref EnvName, '-yourtopic-yourtopic.fifo' ] ]
+      Protocol: 'sqs'
+      Endpoint: !GetAtt yourtopicyourtopicfifoEventsQueue.Arn
+  yourtopicyourtopicfifoEventsQueue:
+    Metadata:
+      'aws:copilot:description': 'A SQS FIFO queue to buffer messages from the topic yourtopic.fifo'
+    Type: AWS::SQS::Queue
+    Properties:
+      KmsMasterKeyId: !Ref EventsKMSKey
+      FifoQueue: true
+      FifoThroughputLimit: perMessageGroupId
+      DeduplicationScope: messageGroup
+      ContentBasedDeduplication: true
+  yourtopicyourtopicfifoQueuePolicy:
+    Type: AWS::SQS::QueuePolicy
+    Properties:
+      Queues: [ !Ref 'yourtopicyourtopicfifoEventsQueue' ]
+      PolicyDocument:
+        Version: '2012-10-17'
+        Statement:
+          - Effect: Allow
+            Principal:
+              AWS:
+                - !GetAtt TaskRole.Arn
+            Action:
+              - sqs:ReceiveMessage
+              - sqs:DeleteMessage
+            Resource: !GetAtt yourtopicyourtopicfifoEventsQueue.Arn
+          - Effect: Allow
+            Principal:
+              Service: sns.amazonaws.com
+            Action:
+              - sqs:SendMessage
+            Resource: !GetAtt yourtopicyourtopicfifoEventsQueue.Arn
+            Condition:
+              ArnEquals:
+                aws:SourceArn: !Join [ '', [ !Sub 'arn:${AWS::Partition}:sns:${AWS::Region}:${AWS::AccountId}:', !Ref AppName, '-', !Ref EnvName, '-yourtopic-yourtopic.fifo' ] ]
+  nonfifotopicnonfifotopicSNSTopicSubscription:
+    Metadata:
+      'aws:copilot:description': 'A SNS subscription to topic nonfifotopic from service nonfifotopic'
+    Type: AWS::SNS::Subscription
+    Properties:
+      TopicArn: !Join [ '', [ !Sub 'arn:${AWS::Partition}:sns:${AWS::Region}:${AWS::AccountId}:', !Ref AppName, '-', !Ref EnvName, '-nonfifotopic-nonfifotopic' ] ]
+      Protocol: 'sqs'
+      Endpoint: !GetAtt nonfifotopicnonfifotopicEventsQueue.Arn
+  nonfifotopicnonfifotopicEventsQueue:
+    Metadata:
+      'aws:copilot:description': 'A SQS queue to buffer messages from the topic nonfifotopic'
+    Type: AWS::SQS::Queue
+    Properties:
+      KmsMasterKeyId: !Ref EventsKMSKey
+  nonfifotopicnonfifotopicQueuePolicy:
+    Type: AWS::SQS::QueuePolicy
+    Properties:
+      Queues: [ !Ref 'nonfifotopicnonfifotopicEventsQueue' ]
+      PolicyDocument:
+        Version: '2012-10-17'
+        Statement:
+          - Effect: Allow
+            Principal:
+              AWS:
+                - !GetAtt TaskRole.Arn
+            Action:
+              - sqs:ReceiveMessage
+              - sqs:DeleteMessage
+            Resource: !GetAtt nonfifotopicnonfifotopicEventsQueue.Arn
+          - Effect: Allow
+            Principal:
+              Service: sns.amazonaws.com
+            Action:
+              - sqs:SendMessage
+            Resource: !GetAtt nonfifotopicnonfifotopicEventsQueue.Arn
+            Condition:
+              ArnEquals:
+                aws:SourceArn: !Join [ '', [ !Sub 'arn:${AWS::Partition}:sns:${AWS::Region}:${AWS::AccountId}:', !Ref AppName, '-', !Ref EnvName, '-nonfifotopic-nonfifotopic' ] ]
   givesOtherdogsSNSTopic:
     Metadata:
       'aws:copilot:description': 'A SNS topic to broadcast givesOtherdogs events'

internal/pkg/deploy/cloudformation/stack/transformers.go

@@ -29,6 +29,12 @@ const (
 	disabled = "DISABLED"
 )
 
+// SQS Queue field values.
+const (
+	sqsDedupeScopeMessageGroup              = "messageGroup"
+	sqsFIFOThroughputLimitPerMessageGroupId = "perMessageGroupId"
+)
+
 // Default values for EFS options
 const (
 	defaultRootDirectory   = "/"
@@ -799,7 +805,6 @@ func convertPublish(topics []manifest.Topic, accountID, region, app, env, svc st
 	for _, topic := range topics {
 		publishers.Topics = append(publishers.Topics, &template.Topic{
 			Name:      topic.Name,
-			Type:      aws.StringValue(topic.Type),
 			AccountID: accountID,
 			Partition: partition.ID(),
 			Region:    region,
@@ -812,19 +817,19 @@ func convertPublish(topics []manifest.Topic, accountID, region, app, env, svc st
 	return &publishers, nil
 }
 
-func convertSubscribe(s manifest.SubscribeConfig) (*template.SubscribeOpts, error) {
-	if s.Topics == nil {
+func convertSubscribe(s *manifest.WorkerService) (*template.SubscribeOpts, error) {
+	if s.Subscribe.Topics == nil {
 		return nil, nil
 	}
 	var subscriptions template.SubscribeOpts
-	for _, sb := range s.Topics {
+	for _, sb := range s.Subscriptions() {
 		ts, err := convertTopicSubscription(sb)
 		if err != nil {
 			return nil, err
 		}
 		subscriptions.Topics = append(subscriptions.Topics, ts)
 	}
-	subscriptions.Queue = convertQueue(s.Queue)
+	subscriptions.Queue = convertQueue(s.Subscribe.Queue)
 	return &subscriptions, nil
 }
 
@@ -861,16 +866,39 @@ func convertFilterPolicy(filterPolicy map[string]interface{}) (*string, error) {
 	return aws.String(string(bytes)), nil
 }
 
-func convertQueue(q manifest.SQSQueue) *template.SQSQueue {
-	if q.IsEmpty() {
+func convertQueue(in manifest.SQSQueue) *template.SQSQueue {
+	if in.IsEmpty() {
 		return nil
 	}
-	return &template.SQSQueue{
-		Retention:  convertRetention(q.Retention),
-		Delay:      convertDelay(q.Delay),
-		Timeout:    convertTimeout(q.Timeout),
-		DeadLetter: convertDeadLetter(q.DeadLetter),
+
+	queue := &template.SQSQueue{
+		Retention:  convertRetention(in.Retention),
+		Delay:      convertDelay(in.Delay),
+		Timeout:    convertTimeout(in.Timeout),
+		DeadLetter: convertDeadLetter(in.DeadLetter),
+	}
+
+	if !in.FIFO.IsEnabled() {
+		return queue
+	}
+
+	if aws.BoolValue(in.FIFO.Enable) {
+		queue.FIFOQueueConfig = &template.FIFOQueueConfig{}
+		return queue
+	}
+
+	if !in.FIFO.Advanced.IsEmpty() {
+		queue.FIFOQueueConfig = &template.FIFOQueueConfig{
+			ContentBasedDeduplication: in.FIFO.Advanced.ContentBasedDeduplication,
+			DeduplicationScope:        in.FIFO.Advanced.DeduplicationScope,
+			FIFOThroughputLimit:       in.FIFO.Advanced.FIFOThroughputLimit,
+		}
+		if aws.BoolValue(in.FIFO.Advanced.HighThroughputFifo) {
+			queue.FIFOQueueConfig.FIFOThroughputLimit = aws.String(sqsFIFOThroughputLimitPerMessageGroupId)
+			queue.FIFOQueueConfig.DeduplicationScope = aws.String(sqsDedupeScopeMessageGroup)
+		}
 	}
+	return queue
 }
 
 func convertTime(t *time.Duration) *int64 {