feat: omg! add queue processing fargate service

.github/workflows/test.yml

@@ -42,6 +42,6 @@ jobs:
       - run: npm ci
       - run: npm test
         env:
-          AWS_REGION: 'us-east-2'
+          AWS_REGION: 'us-east-1'
           AWS_ACCESS_KEY_ID: 'NOSUCH'
           AWS_SECRET_ACCESS_KEY: 'NOSUCH'

cdk.context.json

@@ -0,0 +1,10 @@
+{
+  "availability-zones:account=951776350275:region=us-east-1": [
+    "us-east-1a",
+    "us-east-1b",
+    "us-east-1c",
+    "us-east-1d",
+    "us-east-1e",
+    "us-east-1f"
+  ]
+}

stacks/PickupStack.ts

@@ -0,0 +1,43 @@
+import { PinningServiceStack } from './PinningServiceStack'
+import { StackContext, use, Queue } from '@serverless-stack/resources'
+import { SymlinkFollowMode } from 'aws-cdk-lib'
+import * as ecs from 'aws-cdk-lib/aws-ecs'
+// import * as ecsPatterns from 'aws-cdk-lib/aws-ecs-patterns'
+import { QueueProcessingFargateService } from './lib/queue-processing-fargate-service'
+
+export function PickupStack ({ stack }: StackContext): void {
+  const pinService = use(PinningServiceStack) as unknown as { queue: Queue }
+  // https://docs.aws.amazon.com/cdk/v2/guide/ecs_example.html
+
+  // https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.aws_ecs_patterns-readme.html#queue-processing-services
+  const service = new QueueProcessingFargateService(stack, 'Service', {
+    // https://docs.aws.amazon.com/cdk/v2/guide/assets.html
+    image: ecs.ContainerImage.fromAsset(new URL('../../pickup', import.meta.url).pathname, {
+      // todo: remove me
+      followSymlinks: SymlinkFollowMode.ALWAYS
+    }),
+    containerName: 'pickup',
+    maxScalingCapacity: 2,
+    cpu: 512,
+    memoryLimitMiB: 1024,
+    ephemeralStorageGiB: 64, // max 200
+    // cpu: 4096,
+    // memoryLimitMiB: 8192,
+    environment: {
+      SQS_QUEUE_URL: pinService.queue.queueUrl,
+      GATEWAY_URL: 'http://127.0.0.1:8080'
+    },
+    queue: pinService.queue.cdk.queue
+    // retentionPeriod: Duration.days(1),
+    // visibilityTimeout: Duration.minutes(5),
+  })
+
+  // go-ipfs as sidecar!
+  // see: https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.aws_ecs_patterns-readme.html#deploy-application-and-metrics-sidecar
+  service.taskDefinition.addContainer('ipfs', {
+    image: ecs.ContainerImage.fromRegistry('ipfs/go-ipfs:v0.13.0')
+    // environment: {
+    //   IPFS_PATH: '/data/ipfs'
+    // }
+  })
+}

stacks/PinningServiceStack.ts

@@ -1,9 +1,9 @@
-import { StackContext, Api, Table, Topic } from '@serverless-stack/resources'
+import { StackContext, Api, Table, Queue } from '@serverless-stack/resources'
 
-export function PinningServiceStack ({ stack }: StackContext): void {
-  const topic = new Topic(stack, 'Pin')
+export function PinningServiceStack ({ stack }: StackContext): { table: Table, queue: Queue } {
+  const queue = new Queue(stack, 'Pin')
 
-  const table = new Table(stack, 'PinStatusv2', {
+  const table = new Table(stack, 'PinStatusv4', {
     fields: {
       requestid: 'string',
       userid: 'string'
@@ -18,10 +18,10 @@ export function PinningServiceStack ({ stack }: StackContext): void {
     cors: true,
     defaults: {
       function: {
-        permissions: [table, topic], // Allow the API to access the table and topic
+        permissions: [table, queue], // Allow the API to access the table and topic
         environment: {
           TABLE_NAME: table.tableName,
-          TOPIC_ARN: topic.topicArn
+          QUEUE_URL: queue.queueUrl
         }
       }
     },
@@ -38,7 +38,11 @@ export function PinningServiceStack ({ stack }: StackContext): void {
   // Show the endpoint in the output
   stack.addOutputs({
     ApiEndpoint: api.url,
-    TopicName: topic.topicName,
-    TopicARN: topic.topicArn
+    QueueURL: queue.queueUrl
   })
+
+  return {
+    table,
+    queue
+  }
 }

stacks/index.ts

@@ -1,3 +1,4 @@
+import { PickupStack } from './PickupStack'
 import { PinningServiceStack } from './PinningServiceStack'
 import { App } from '@serverless-stack/resources'
 
@@ -10,4 +11,5 @@ export default function (app: App): void {
     }
   })
   app.stack(PinningServiceStack)
+  app.stack(PickupStack)
 }

stacks/lib/queue-processing-fargate-service.ts

@@ -0,0 +1,178 @@
+// Forked from https://github.com/aws/aws-cdk/blob/f2b4effc903ab3a36dc925516f3329f236d03a70/packages/%40aws-cdk/aws-ecs-patterns/lib/fargate/queue-processing-fargate-service.ts
+// Adds `ephemeralStorageGiB` property to the Service, passed thru to FargateTaskDefinition
+// Official support is stuck in a stalled PR here https://github.com/aws/aws-cdk/pull/18106
+import * as ec2 from 'aws-cdk-lib/aws-ec2'
+import { FargatePlatformVersion, FargateService, FargateTaskDefinition, HealthCheck } from 'aws-cdk-lib/aws-ecs'
+import { Construct } from 'constructs'
+import { QueueProcessingServiceBase, QueueProcessingServiceBaseProps } from 'aws-cdk-lib/aws-ecs-patterns'
+
+/**
+ * The properties for the QueueProcessingFargateService service.
+ */
+export interface QueueProcessingFargateServiceProps extends QueueProcessingServiceBaseProps {
+  /**
+   * The number of cpu units used by the task.
+   *
+   * Valid values, which determines your range of valid values for the memory parameter:
+   *
+   * 256 (.25 vCPU) - Available memory values: 0.5GB, 1GB, 2GB
+   *
+   * 512 (.5 vCPU) - Available memory values: 1GB, 2GB, 3GB, 4GB
+   *
+   * 1024 (1 vCPU) - Available memory values: 2GB, 3GB, 4GB, 5GB, 6GB, 7GB, 8GB
+   *
+   * 2048 (2 vCPU) - Available memory values: Between 4GB and 16GB in 1GB increments
+   *
+   * 4096 (4 vCPU) - Available memory values: Between 8GB and 30GB in 1GB increments
+   *
+   * This default is set in the underlying FargateTaskDefinition construct.
+   *
+   * @default 256
+   */
+  readonly cpu?: number
+
+  /**
+   * The amount (in MiB) of memory used by the task.
+   *
+   * This field is required and you must use one of the following values, which determines your range of valid values
+   * for the cpu parameter:
+   *
+   * 0.5GB, 1GB, 2GB - Available cpu values: 256 (.25 vCPU)
+   *
+   * 1GB, 2GB, 3GB, 4GB - Available cpu values: 512 (.5 vCPU)
+   *
+   * 2GB, 3GB, 4GB, 5GB, 6GB, 7GB, 8GB - Available cpu values: 1024 (1 vCPU)
+   *
+   * Between 4GB and 16GB in 1GB increments - Available cpu values: 2048 (2 vCPU)
+   *
+   * Between 8GB and 30GB in 1GB increments - Available cpu values: 4096 (4 vCPU)
+   *
+   * This default is set in the underlying FargateTaskDefinition construct.
+   *
+   * @default 512
+   */
+  readonly memoryLimitMiB?: number
+
+  /**
+   * The amount (in GiB) of ephemeral storage to be allocated to the task. The maximum supported value is 200 GiB.
+   *
+   * NOTE: This parameter is only supported for tasks hosted on AWS Fargate using platform version 1.4.0 or later.
+   *
+   * This default is set in the underlying FargateTaskDefinition construct.
+   *
+   * @default 20
+   */
+  readonly ephemeralStorageGiB?: number
+
+  /**
+   * The platform version on which to run your service.
+   *
+   * If one is not specified, the LATEST platform version is used by default. For more information, see
+   * [AWS Fargate Platform Versions](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/platform_versions.html)
+   * in the Amazon Elastic Container Service Developer Guide.
+   *
+   * @default Latest
+   */
+  readonly platformVersion?: FargatePlatformVersion
+
+  /**
+   * Optional name for the container added
+   *
+   * @default - QueueProcessingContainer
+   */
+  readonly containerName?: string
+
+  /**
+   * The health check command and associated configuration parameters for the container.
+   *
+   * @default - Health check configuration from container.
+   */
+  readonly healthCheck?: HealthCheck
+
+  /**
+   * The subnets to associate with the service.
+   *
+   * @default - Public subnets if `assignPublicIp` is set, otherwise the first available one of Private, Isolated, Public, in that order.
+   */
+  readonly taskSubnets?: ec2.SubnetSelection
+
+  /**
+   * The security groups to associate with the service. If you do not specify a security group, a new security group is created.
+   *
+   * @default - A new security group is created.
+   */
+  readonly securityGroups?: ec2.ISecurityGroup[]
+
+  /**
+   * Specifies whether the task's elastic network interface receives a public IP address.
+   *
+   * If true, each task will receive a public IP address.
+   *
+   * @default false
+   */
+  readonly assignPublicIp?: boolean
+}
+
+/**
+ * Class to create a queue processing Fargate service
+ */
+export class QueueProcessingFargateService extends QueueProcessingServiceBase {
+  /**
+   * The Fargate service in this construct.
+   */
+  public readonly service: FargateService
+  /**
+   * The Fargate task definition in this construct.
+   */
+  public readonly taskDefinition: FargateTaskDefinition
+
+  /**
+   * Constructs a new instance of the QueueProcessingFargateService class.
+   */
+  constructor (scope: Construct, id: string, props: QueueProcessingFargateServiceProps & {ephemeralStorageGiB: number}) {
+    super(scope, id, props)
+
+    // Create a Task Definition for the container to start
+    this.taskDefinition = new FargateTaskDefinition(this, 'QueueProcessingTaskDef', {
+      memoryLimitMiB: props.memoryLimitMiB ?? 512,
+      cpu: props.cpu ?? 256,
+      family: props.family,
+      ephemeralStorageGiB: props.ephemeralStorageGiB
+    })
+
+    const containerName = props.containerName ?? 'QueueProcessingContainer'
+
+    this.taskDefinition.addContainer(containerName, {
+      image: props.image,
+      command: props.command,
+      environment: this.environment,
+      secrets: this.secrets,
+      logging: this.logDriver,
+      healthCheck: props.healthCheck
+    })
+
+    // Create a Fargate service with the previously defined Task Definition and configure
+    // autoscaling based on cpu utilization and number of messages visible in the SQS queue.
+    this.service = new FargateService(this, 'QueueProcessingFargateService', {
+      cluster: this.cluster,
+      // desiredCount: desiredCount,
+      taskDefinition: this.taskDefinition,
+      serviceName: props.serviceName,
+      minHealthyPercent: props.minHealthyPercent,
+      maxHealthyPercent: props.maxHealthyPercent,
+      propagateTags: props.propagateTags,
+      enableECSManagedTags: props.enableECSManagedTags,
+      platformVersion: props.platformVersion,
+      deploymentController: props.deploymentController,
+      securityGroups: props.securityGroups,
+      vpcSubnets: props.taskSubnets,
+      assignPublicIp: props.assignPublicIp,
+      circuitBreaker: props.circuitBreaker,
+      capacityProviderStrategies: props.capacityProviderStrategies,
+      // enableExecuteCommand: props.enableExecuteCommand
+    })
+
+    this.configureAutoscalingForService(this.service)
+    this.grantPermissionsToService(this.service)
+  }
+}

tsconfig.json

@@ -1,4 +1,8 @@
 {
   "extends": "@tsconfig/node16/tsconfig.json",
-  "include": ["stacks/**/*"]
+  "include": ["stacks/**/*"],
+  "compilerOptions": {
+    "module": "esnext",
+    "moduleResolution": "node",
+  }
 }