feat(terraform): add backups for Postgres service (#292)

* feat(terraform): add backup container with default to EFS storage

* feat(terraform): ability to prevent EFS destroy

* feat(terraform): ability to prevent volume destroy

* feat(terraform): prevent backup destroy

* refactor(terraform): improve app config override

* fix(terraform): set default db to api

* feat(terraform): ability to backup volumes

* feat(terraform): ability to backup mount point

* fix(terraform): remove unused envs from backups

* fix(terraform): connect backup and postgres through private dns

apps/terraform/src/configs/apps.config.ts

@@ -1,7 +1,4 @@
-import { ApiComponentPublicConfig } from '@tractr/terraform-service-api';
-import { PostgresComponentPublicConfig } from '@tractr/terraform-service-postgres';
-import { PwaComponentPublicConfig } from '@tractr/terraform-service-pwa';
-import { ReverseProxyComponentPublicConfig } from '@tractr/terraform-service-reverse-proxy';
+import { Environment } from '../interfaces';
 
 /**
  * @example
@@ -20,10 +17,10 @@ import { ReverseProxyComponentPublicConfig } from '@tractr/terraform-service-rev
  *  },
  * }
  */
-export const ApiConfig: ApiComponentPublicConfig = {};
 
-export const PwaConfig: PwaComponentPublicConfig = {};
-
-export const PostgresConfig: PostgresComponentPublicConfig = {};
-
-export const ReverseProxyConfig: ReverseProxyComponentPublicConfig = {};
+export const AppConfig: Required<Environment['config']> = {
+  api: {},
+  pwa: {},
+  postgres: {},
+  reverseProxy: {},
+};

apps/terraform/src/configs/environments.config.ts

@@ -5,19 +5,24 @@ export const Environments: Environment[] = [
     name: 'Production',
     resourceId: 'prod',
     subDomain: 'www',
-    pwaConfig: { containerConfig: { imageTag: 'production' } },
-    apiConfig: {
-      containerConfig: { imageTag: 'production' },
-      desiredCount: 2,
-      cpu: '512',
-      memory: '1024',
+    config: {
+      pwa: { containerConfig: { imageTag: 'production' } },
+      api: {
+        containerConfig: { imageTag: 'production' },
+        desiredCount: 2,
+        cpu: '512',
+        memory: '1024',
+      },
+      postgres: { enableBackups: true },
     },
   },
   {
     name: 'Staging',
     resourceId: 'staging',
     subDomain: 'staging',
-    pwaConfig: { containerConfig: { imageTag: 'latest' } },
-    apiConfig: { containerConfig: { imageTag: 'latest' } },
+    config: {
+      pwa: { containerConfig: { imageTag: 'latest' } },
+      api: { containerConfig: { imageTag: 'latest' } },
+    },
   },
 ];

apps/terraform/src/interfaces/environment.interface.ts

@@ -1,5 +1,7 @@
 import { ApiComponentPublicConfig } from '@tractr/terraform-service-api';
+import { PostgresComponentPublicConfig } from '@tractr/terraform-service-postgres';
 import { PwaComponentPublicConfig } from '@tractr/terraform-service-pwa';
+import { ReverseProxyComponentPublicConfig } from '@tractr/terraform-service-reverse-proxy';
 
 export interface Environment {
   /**
@@ -14,12 +16,26 @@ export interface Environment {
    * Subdomain that will host this environment
    */
   subDomain: string;
+
   /**
-   * PWA config override
-   */
-  pwaConfig: PwaComponentPublicConfig;
-  /**
-   * API config override
+   * Configs that override the main configs
    */
-  apiConfig: ApiComponentPublicConfig;
+  config: {
+    /**
+     * PWA config override
+     */
+    pwa?: PwaComponentPublicConfig;
+    /**
+     * API config override
+     */
+    api?: ApiComponentPublicConfig;
+    /**
+     * Postgres config override
+     */
+    postgres?: PostgresComponentPublicConfig;
+    /**
+     * Reverse proxy config override
+     */
+    reverseProxy?: ReverseProxyComponentPublicConfig;
+  };
 }

apps/terraform/src/stacks/main.stack.ts

@@ -1,13 +1,7 @@
 import { Construct } from 'constructs';
 import * as deepmerge from 'deepmerge';
 
-import {
-  ApiConfig,
-  Environments,
-  PostgresConfig,
-  PwaConfig,
-  ReverseProxyConfig,
-} from '../configs';
+import { AppConfig, Environments } from '../configs';
 import { TerraformEnvironmentVariables } from '../dtos';
 
 import { AwsStack, AwsStackConfig } from '@tractr/terraform-aws-stack';
@@ -59,12 +53,15 @@ export class MainStack extends AwsStack<AwsStackConfig> {
 
     // Create a pool for each environment
     for (const environment of Environments) {
+      // Merge app configs and environment configs
+      const mergedConfig = deepmerge(AppConfig, environment.config);
+
       // Add the pool group that will host our container
       const poolGroup = new PoolGroup(this, environment.resourceId, {
         registryGroup: this.registryGroup,
         networkGroup: this.networkGroup,
         zoneGroup: this.zoneGroup,
-        reverseProxyConfig: ReverseProxyConfig,
+        reverseProxyConfig: mergedConfig.reverseProxy,
         subDomain: environment.subDomain,
         ownerPictureConfig: {
           s3PublicRead: true,
@@ -73,23 +70,21 @@ export class MainStack extends AwsStack<AwsStackConfig> {
       });
 
       // Add a pwa as a http service
-      const mergedPwaConfig = deepmerge(PwaConfig, environment.pwaConfig);
-      poolGroup.addHttpService(PwaComponent, 'pwa', mergedPwaConfig);
+      poolGroup.addHttpService(PwaComponent, 'pwa', mergedConfig.pwa);
 
       // Add a api as a http service
-      const mergedApiConfig = deepmerge(ApiConfig, environment.apiConfig);
       const api = poolGroup.addHttpService(
         ApiComponent,
         'api',
-        mergedApiConfig,
+        mergedConfig.api,
       );
 
       // Add a postgres as a backend service
       poolGroup.addBackendService(
         PostgresComponent,
         'postgres',
         [api],
-        PostgresConfig,
+        mergedConfig.postgres,
       );
 
       // Store group

libs/terraform/component/volume/README.md

@@ -10,6 +10,7 @@ const volume = new VolumeComponent(this, 'vol', {
   vpcId: 'xxxxxxxx',
   subnetsIds: ['aaaaa', 'bbbbb'],
   clientsSecurityGroupsIds: ['ssssss'],
+  preventDestroy: false,
 });
 ```
 

libs/terraform/component/volume/src/lib/volume.component.ts

@@ -1,4 +1,5 @@
 import {
+  EfsBackupPolicy,
   EfsFileSystem,
   EfsMountTarget,
   SecurityGroup,
@@ -23,6 +24,8 @@ export interface VolumeComponentConfig extends ConstructOptions {
     | 'AFTER_30_DAYS'
     | 'AFTER_60_DAYS'
     | 'AFTER_90_DAYS';
+  preventDestroy?: boolean;
+  enableBackups?: boolean;
 }
 
 export class VolumeComponent<
@@ -32,12 +35,17 @@ export class VolumeComponent<
 
   protected readonly efsFileSystem: EfsFileSystem;
 
+  protected readonly efsBackupPolicy: EfsBackupPolicy | undefined;
+
   protected readonly efsMountTargets: EfsMountTarget[];
 
   constructor(scope: AwsProviderConstruct, id: string, config: T) {
     super(scope, id, config);
     this.securityGroup = this.createSecurityGroup();
     this.efsFileSystem = this.createEfsFileSystem();
+    if (this.config.enableBackups) {
+      this.efsBackupPolicy = this.createEfsBackupPolicy();
+    }
     this.efsMountTargets = this.createEfsMountTargets();
   }
 
@@ -65,10 +73,18 @@ export class VolumeComponent<
       lifecyclePolicy: [
         { transitionToIa: this.config.transitionToIa || 'AFTER_90_DAYS' },
       ],
+      lifecycle: { preventDestroy: !!this.config.preventDestroy },
       tags: this.getResourceNameAsTag('fs'),
     });
   }
 
+  protected createEfsBackupPolicy() {
+    return new EfsBackupPolicy(this, 'bck', {
+      fileSystemId: this.getFileSystemIdAsToken(),
+      backupPolicy: [{ status: 'ENABLED' }],
+    });
+  }
+
   protected createEfsMountTargets() {
     return this.config.subnetsIds.map((subnetId, index) =>
       this.createEfsMountTarget(subnetId, index),

libs/terraform/service/ecs/src/lib/interfaces/container.interface.ts

@@ -23,7 +23,7 @@ export interface ContainerDefinition {
     hostPort?: number;
     protocol?: 'tcp' | 'udp';
   }[];
-  mountPoints?: MountPoint[];
+  mountPoints?: MountPointDefinition[];
   dockerLabels?: Record<string, string>;
 }
 
@@ -42,11 +42,16 @@ export interface ImageDefinition {
   imageUri: string;
 }
 
-export interface MountPoint {
+export interface MountPointDefinition {
   sourceVolume: string;
   containerPath: string;
 }
 
+export interface MountPoint extends MountPointDefinition {
+  preventDestroy?: boolean;
+  enableBackups?: boolean;
+}
+
 export type EnvironmentCb<C extends ContainerConfig = ContainerConfig> = (
   service: ServiceComponent,
   config: C,

libs/terraform/service/ecs/src/lib/interfaces/services.interface.ts

@@ -1,5 +1,7 @@
 import { ConstructOptions } from 'constructs';
 
+import { Container } from '../containers';
+
 import { DockerApplications } from '@tractr/terraform-group-registry';
 
 // Check cpu/memory pairs: https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-cpu-memory-error.html
@@ -17,6 +19,13 @@ export type MemoryValue =
   | '16384'
   | '30720';
 
+export type VolumesConfig = {
+  preventDestroy: boolean;
+  enableBackups: boolean;
+  containers: Container[];
+};
+export type VolumesConfigs = Record<string, VolumesConfig>;
+
 export interface ServiceComponentInternalConfig extends ConstructOptions {
   vpcId: string;
   subnetsIds: string[];

libs/terraform/service/ecs/src/lib/services/backend-service.component.ts

@@ -30,9 +30,18 @@ export abstract class BackendServiceComponent<
         fromPort: port,
         toPort: port,
         securityGroups: this.config.clientsSecurityGroupsIds,
+        selfAttribute: this.shouldAccessItself(),
       })),
     };
   }
 
+  /**
+   * Whether to add `self` attribute on security group ingress rules in order to allow this service to call itself.
+   * For exemple, a service with two different containers that need to call each other.
+   */
+  protected shouldAccessItself(): boolean {
+    return false;
+  }
+
   protected abstract getIngressPorts(): number[];
 }

libs/terraform/service/ecs/src/lib/services/service.component.ts

@@ -17,6 +17,8 @@ import { Container } from '../containers';
 import {
   ServiceComponentConfig,
   ServiceComponentDefaultConfig,
+  VolumesConfig,
+  VolumesConfigs,
 } from '../interfaces';
 
 import {
@@ -53,7 +55,7 @@ export abstract class ServiceComponent<
 
   protected readonly containers: Container[];
 
-  protected readonly volumesNames: string[];
+  protected readonly volumesConfigs: VolumesConfigs;
 
   protected readonly volumeComponentsMap:
     | Record<string, VolumeComponent>
@@ -68,7 +70,7 @@ export abstract class ServiceComponent<
     this.config = deepmerge(this.getDefaultConfig(), config) as C & D;
 
     this.containers = this.getContainers();
-    this.volumesNames = this.getVolumesNames();
+    this.volumesConfigs = this.getVolumesConfigs();
     this.securityGroup = this.createSecurityGroup();
     if (this.shouldCreateVolumeComponent()) {
       this.volumeComponentsMap = this.createVolumeComponentsMap();
@@ -193,39 +195,65 @@ export abstract class ServiceComponent<
     };
   }
 
-  protected getVolumesNames(): string[] {
-    const volumes = new Set<string>();
-    this.containers.forEach((container) => {
-      container.getMountPoints().forEach((mountPoint) => {
-        volumes.add(mountPoint.sourceVolume);
-      });
-    });
-    return [...volumes];
+  /**
+   * Aggregate and group all mounts points across every container and merge boolean properties
+   * @protected
+   */
+  protected getVolumesConfigs(): VolumesConfigs {
+    const volumes: VolumesConfigs = {};
+    for (const container of this.containers) {
+      const mountPoints = container.getMountPoints();
+      for (const mountPoint of mountPoints) {
+        const name = mountPoint.sourceVolume;
+        const previous: VolumesConfig = volumes[name]
+          ? volumes[name]
+          : {
+              containers: [],
+              preventDestroy: false,
+              enableBackups: false,
+            };
+
+        volumes[name] = {
+          containers: [...previous.containers, container],
+          preventDestroy: mountPoint.preventDestroy || previous.preventDestroy,
+          enableBackups: mountPoint.enableBackups || previous.enableBackups,
+        };
+      }
+    }
+    return volumes;
   }
 
   protected shouldCreateVolumeComponent() {
-    return this.volumesNames.length > 0;
+    return Object.keys(this.volumesConfigs).length > 0;
   }
 
   protected createVolumeComponentsMap() {
-    return this.volumesNames.reduce(
-      (map, name) => ({
+    return Object.entries(this.volumesConfigs).reduce(
+      (map, [name, config]) => ({
         ...map,
-        [name]: this.createVolumeComponent(name),
+        [name]: this.createVolumeComponent(name, config),
       }),
       {} as Record<string, VolumeComponent>,
     );
   }
 
-  protected createVolumeComponent(name: string) {
-    return new VolumeComponent(this, name, this.getVolumeComponentConfig());
+  protected createVolumeComponent(name: string, config: VolumesConfig) {
+    return new VolumeComponent(
+      this,
+      name,
+      this.getVolumeComponentConfig(config),
+    );
   }
 
-  protected getVolumeComponentConfig(): VolumeComponentConfig {
+  protected getVolumeComponentConfig(
+    config: VolumesConfig,
+  ): VolumeComponentConfig {
     return {
       vpcId: this.config.vpcId,
       subnetsIds: this.config.subnetsIds,
       clientsSecurityGroupsIds: [this.getSecurityGroupIdAsToken()],
+      preventDestroy: config.preventDestroy,
+      enableBackups: config.enableBackups,
     };
   }