AWS CDK Fargate deployed Grimoire Lab.

aws/.gitignore

@@ -0,0 +1,9 @@
+*.js
+!jest.config.js
+*.d.ts
+node_modules
+
+# CDK asset staging directory
+.cdk.staging
+cdk.out
+cdk.context.json

aws/.npmignore

@@ -0,0 +1,6 @@
+*.ts
+!*.d.ts
+
+# CDK asset staging directory
+.cdk.staging
+cdk.out

aws/README.md

@@ -0,0 +1,84 @@
+# Running GrimoireLab with AWS Fargate
+
+In this folder are configuration files for deploying GrimoireLab using [AWS](https://aws.amazon.com). You will need a functional AWS account and you should be able to deploy that AWS account. This setup uses [AWS Fargate](https://aws.amazon.com/fargate) with a Network Load Balancer.
+
+## Requirements
+
+- [Node.js 20.10.0+](https://nodejs.org/en/download)
+- [AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-welcome.html)
+- [AWS credentials configured](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-configure.html)
+- Docker Images built for GrimoireLab (Mordred), NGINX, Sorting Hat Working, Sorting Hat, MariaDB, Redis, OpenSearch, and OpenSearch Dashboards.
+
+## Limitations
+
+- This setup relies on all configuration files (e.g., default-grimoirelab-settings/projects.json) being included in the Docker Images. This configuration has not been setup for EFS volume mounts or the use of AWS S3.
+
+## Getting Started
+
+### Install
+
+```bash
+cd aws
+npm install
+npm run build
+```
+
+### Setup environmental variables for your account and region
+
+| Variable             | Value                                       |
+| -------------------- | ------------------------------------------- |
+| GRIMOIRE_AWS_ACCOUNT | Your AWS account                            |
+| GRIMOIRE_AWS_REGION  | Your preferred AWS region (e.g., eu-west-2) |
+
+### Bootstrap CDK
+
+```bash
+npx cdk bootstrap aws://${GRIMOIRE_AWS_ACCOUNT}/${GRIMOIRE_AWS_REGION}
+```
+
+### Docker Images
+
+#### Setup your AWS registry
+
+```bash
+npx cdk deploy AwsEcrStack
+```
+
+#### Login to your AWS registry
+
+```bash
+aws ecr get-login-password --region ${GRIMOIRE_AWS_REGION} | docker login --username AWS --password-stdin ${GRIMOIRE_AWS_ACCOUNT}.dkr.ecr.${GRIMOIRE_AWS_REGION}.amazonaws.com/grimoirelab
+```
+
+#### Push Images to your AWS registry
+
+```bash
+docker push ${GRIMOIRE_AWS_ACCOUNT}.dkr.ecr.${GRIMOIRE_AWS_REGION}.amazonaws.com/grimoirelab:latest
+docker push ${GRIMOIRE_AWS_ACCOUNT}.dkr.ecr.${GRIMOIRE_AWS_REGION}.amazonaws.com/nginx:latest
+docker push ${GRIMOIRE_AWS_ACCOUNT}.dkr.ecr.${GRIMOIRE_AWS_REGION}.amazonaws.com/sortinghat-working:latest
+docker push ${GRIMOIRE_AWS_ACCOUNT}.dkr.ecr.${GRIMOIRE_AWS_REGION}.amazonaws.com/sortinghat:latest
+docker push ${GRIMOIRE_AWS_ACCOUNT}.dkr.ecr.${GRIMOIRE_AWS_REGION}.amazonaws.com/mariadb:latest
+docker push ${GRIMOIRE_AWS_ACCOUNT}.dkr.ecr.${GRIMOIRE_AWS_REGION}.amazonaws.com/redis:latest
+docker push ${GRIMOIRE_AWS_ACCOUNT}.dkr.ecr.${GRIMOIRE_AWS_REGION}.amazonaws.com/opensearch:1.3.7
+docker push ${GRIMOIRE_AWS_ACCOUNT}.dkr.ecr.${GRIMOIRE_AWS_REGION}.amazonaws.com/opensearch-dashboards:1.3.7
+```
+
+### Deploy the AWS stacks
+
+```bash
+npx cdk deploy AwsVpcStack
+npx cdk deploy AwsEcsRedisStack
+npx cdk deploy AwsEcsOpenSearchStack
+npx cdk deploy AwsEcsSortingHatStack
+npx cdk deploy AwsEcsMordredStack
+npx cdk deploy AwsNlbStack
+```
+
+## Test the AWS stack deployment
+
+Visit the AWS Load balancers website, select grimoire-nlb, and visit the DNS name in your browser (e.g., `grimoire-nlb-2ffec2c2fc78664d.elb.eu-west-2.amazonaws.com`).
+
+## Improvements
+
+1. Setup RDS in place of mariadb service.
+2. Setup EFS to store configuration files and store indices.

aws/bin/aws.ts

@@ -0,0 +1,91 @@
+#!/usr/bin/env node
+import "source-map-support/register";
+import * as cdk from "aws-cdk-lib";
+import { AwsVpcStack } from "../lib/aws-vpc-stack";
+import { AwsEcrStack } from "../lib/aws-ecr-stack";
+import { AwsEcsRedisStack } from "../lib/aws-ecs-redis-stack";
+import { AwsEcsOpenSearchStack } from "../lib/aws-ecs-opensearch-stack";
+import { AwsEcsMordredStack } from "../lib/aws-ecs-mordred-stack";
+import { AwsEcsSortingHatStack } from "../lib/aws-ecs-sortinghat-stack";
+import { AwsNlbStack } from "../lib/aws-nlb-stack";
+
+const app = new cdk.App();
+
+//
+// Environment variables
+// Should be set in local environment or workflows before running `cdk` commands
+const { GRIMOIRE_AWS_ACCOUNT, GRIMOIRE_AWS_REGION } = process.env;
+
+//
+// ECR Repository Stack
+// Create ECR repository separated out of main stack to avoid unneccessary deletion
+new AwsEcrStack(app, "AwsEcrStack", {
+  env: { account: `${GRIMOIRE_AWS_ACCOUNT}`, region: `${GRIMOIRE_AWS_REGION}` },
+});
+
+//
+// VPC Stack separated out of main stack to avoid unneccessary deletion
+const vpcStack = new AwsVpcStack(app, "AwsVpcStack", {
+  env: { account: `${GRIMOIRE_AWS_ACCOUNT}`, region: `${GRIMOIRE_AWS_REGION}` },
+});
+
+//
+// ECS Stack with a cluser and tasks for redis
+new AwsEcsRedisStack(app, "AwsEcsRedisStack", {
+  env: { account: `${GRIMOIRE_AWS_ACCOUNT}`, region: `${GRIMOIRE_AWS_REGION}` },
+  vpc: vpcStack.vpc,
+  cluster: vpcStack.cluster,
+  appLogGroup: vpcStack.appLogGroup,
+  securityGroup: vpcStack.securityGroup,
+});
+
+//
+// ECS Stack with a cluser and tasks for opensearch
+const openSearchStack = new AwsEcsOpenSearchStack(
+  app,
+  "AwsEcsOpenSearchStack",
+  {
+    env: {
+      account: `${GRIMOIRE_AWS_ACCOUNT}`,
+      region: `${GRIMOIRE_AWS_REGION}`,
+    },
+    vpc: vpcStack.vpc,
+    cluster: vpcStack.cluster,
+    appLogGroup: vpcStack.appLogGroup,
+    securityGroup: vpcStack.securityGroup,
+  }
+);
+
+//
+// ECS Stack with a cluser and tasks for sortinghat
+new AwsEcsSortingHatStack(app, "AwsEcsSortingHatStack", {
+  env: { account: `${GRIMOIRE_AWS_ACCOUNT}`, region: `${GRIMOIRE_AWS_REGION}` },
+  vpc: vpcStack.vpc,
+  cluster: vpcStack.cluster,
+  appLogGroup: vpcStack.appLogGroup,
+  securityGroup: vpcStack.securityGroup,
+});
+
+//
+// ECS Stack with a cluser and tasks for mordred
+new AwsEcsMordredStack(app, "AwsEcsMordredStack", {
+  env: { account: `${GRIMOIRE_AWS_ACCOUNT}`, region: `${GRIMOIRE_AWS_REGION}` },
+  vpc: vpcStack.vpc,
+  cluster: vpcStack.cluster,
+  appLogGroup: vpcStack.appLogGroup,
+  securityGroup: vpcStack.securityGroup,
+});
+
+//
+// NLB Stack separated out of main stack to avoid unnecessary deletion
+new AwsNlbStack(app, "AwsNlbStack", {
+  env: { account: `${GRIMOIRE_AWS_ACCOUNT}`, region: `${GRIMOIRE_AWS_REGION}` },
+  vpc: vpcStack.vpc,
+  cluster: vpcStack.cluster,
+  appLogGroup: vpcStack.appLogGroup,
+  //
+  // Connect Network Load Balancer to OpenSearch
+  theTaskDefinition: openSearchStack.ecsService4,
+  theTaskDefinitionContainerName: "opensearch-dashboards",
+  theTaskDefinitionPort: 5601,
+});

aws/cdk.json

@@ -0,0 +1,68 @@
+{
+  "app": "npx ts-node --prefer-ts-exts bin/aws.ts",
+  "watch": {
+    "include": [
+      "**"
+    ],
+    "exclude": [
+      "README.md",
+      "cdk*.json",
+      "**/*.d.ts",
+      "**/*.js",
+      "tsconfig.json",
+      "package*.json",
+      "yarn.lock",
+      "node_modules",
+      "test"
+    ]
+  },
+  "context": {
+    "@aws-cdk/aws-lambda:recognizeLayerVersion": true,
+    "@aws-cdk/core:checkSecretUsage": true,
+    "@aws-cdk/core:target-partitions": [
+      "aws",
+      "aws-cn"
+    ],
+    "@aws-cdk-containers/ecs-service-extensions:enableDefaultLogDriver": true,
+    "@aws-cdk/aws-ec2:uniqueImdsv2TemplateName": true,
+    "@aws-cdk/aws-ecs:arnFormatIncludesClusterName": true,
+    "@aws-cdk/aws-iam:minimizePolicies": true,
+    "@aws-cdk/core:validateSnapshotRemovalPolicy": true,
+    "@aws-cdk/aws-codepipeline:crossAccountKeyAliasStackSafeResourceName": true,
+    "@aws-cdk/aws-s3:createDefaultLoggingPolicy": true,
+    "@aws-cdk/aws-sns-subscriptions:restrictSqsDescryption": true,
+    "@aws-cdk/aws-apigateway:disableCloudWatchRole": true,
+    "@aws-cdk/core:enablePartitionLiterals": true,
+    "@aws-cdk/aws-events:eventsTargetQueueSameAccount": true,
+    "@aws-cdk/aws-iam:standardizedServicePrincipals": true,
+    "@aws-cdk/aws-ecs:disableExplicitDeploymentControllerForCircuitBreaker": true,
+    "@aws-cdk/aws-iam:importedRoleStackSafeDefaultPolicyName": true,
+    "@aws-cdk/aws-s3:serverAccessLogsUseBucketPolicy": true,
+    "@aws-cdk/aws-route53-patters:useCertificate": true,
+    "@aws-cdk/customresources:installLatestAwsSdkDefault": false,
+    "@aws-cdk/aws-rds:databaseProxyUniqueResourceName": true,
+    "@aws-cdk/aws-codedeploy:removeAlarmsFromDeploymentGroup": true,
+    "@aws-cdk/aws-apigateway:authorizerChangeDeploymentLogicalId": true,
+    "@aws-cdk/aws-ec2:launchTemplateDefaultUserData": true,
+    "@aws-cdk/aws-secretsmanager:useAttachedSecretResourcePolicyForSecretTargetAttachments": true,
+    "@aws-cdk/aws-redshift:columnId": true,
+    "@aws-cdk/aws-stepfunctions-tasks:enableEmrServicePolicyV2": true,
+    "@aws-cdk/aws-ec2:restrictDefaultSecurityGroup": true,
+    "@aws-cdk/aws-apigateway:requestValidatorUniqueId": true,
+    "@aws-cdk/aws-kms:aliasNameRef": true,
+    "@aws-cdk/aws-autoscaling:generateLaunchTemplateInsteadOfLaunchConfig": true,
+    "@aws-cdk/core:includePrefixInUniqueNameGeneration": true,
+    "@aws-cdk/aws-efs:denyAnonymousAccess": true,
+    "@aws-cdk/aws-opensearchservice:enableOpensearchMultiAzWithStandby": true,
+    "@aws-cdk/aws-lambda-nodejs:useLatestRuntimeVersion": true,
+    "@aws-cdk/aws-efs:mountTargetOrderInsensitiveLogicalId": true,
+    "@aws-cdk/aws-rds:auroraClusterChangeScopeOfInstanceParameterGroupWithEachParameters": true,
+    "@aws-cdk/aws-appsync:useArnForSourceApiAssociationIdentifier": true,
+    "@aws-cdk/aws-rds:preventRenderingDeprecatedCredentials": true,
+    "@aws-cdk/aws-codepipeline-actions:useNewDefaultBranchForCodeCommitSource": true,
+    "@aws-cdk/aws-cloudwatch-actions:changeLambdaPermissionLogicalIdForLambdaAction": true,
+    "@aws-cdk/aws-codepipeline:crossAccountKeysDefaultValueToFalse": true,
+    "@aws-cdk/aws-codepipeline:defaultPipelineTypeToV2": true,
+    "@aws-cdk/aws-kms:reduceCrossAccountRegionPolicyScope": true
+  }
+}

aws/jest.config.js

@@ -0,0 +1,8 @@
+module.exports = {
+  testEnvironment: 'node',
+  roots: ['<rootDir>/test'],
+  testMatch: ['**/*.test.ts'],
+  transform: {
+    '^.+\\.tsx?$': 'ts-jest'
+  }
+};

aws/lib/aws-ecr-stack.ts

@@ -0,0 +1,41 @@
+import * as cdk from "aws-cdk-lib";
+import { Construct } from "constructs";
+import * as ecr from "aws-cdk-lib/aws-ecr";
+
+export class AwsEcrStack extends cdk.Stack {
+  constructor(scope: Construct, id: string, props: cdk.StackProps) {
+    super(scope, id, props);
+
+    new ecr.Repository(this, "grimoire-ecr-repository-nginx", {
+      repositoryName: "nginx",
+    });
+
+    new ecr.Repository(this, "grimoire-ecr-repository-grimoirelab", {
+      repositoryName: "grimoirelab",
+    });
+
+    new ecr.Repository(this, "grimoire-ecr-repository-sortinghat-worker", {
+      repositoryName: "sortinghat-worker",
+    });
+
+    new ecr.Repository(this, "grimoire-ecr-repository-sortinghat", {
+      repositoryName: "sortinghat",
+    });
+
+    new ecr.Repository(this, "grimoire-ecr-repository-grimoire-mariadb", {
+      repositoryName: "mariadb",
+    });
+
+    new ecr.Repository(this, "grimoire-ecr-repository-grimoire-redis", {
+      repositoryName: "redis",
+    });
+
+    new ecr.Repository(this, "grimoire-ecr-repository-grimoire-opensearch", {
+      repositoryName: "opensearch",
+    });
+
+    new ecr.Repository(this, "grimoire-ecr-repository-grimoire-opensearch-dashboards", {
+      repositoryName: "opensearch-dashboards",
+    });
+  }
+}