Add docker cloudwatch config

README.md

@@ -269,6 +269,10 @@ The following inputs can be used as `step.with` keys
 | `docker_repo_app_directory` | String | Relative path for the directory of the app. (i.e. where the `docker-compose.yaml` file is located). This is the directory that is copied into the EC2 instance. Default is `/`, the root of the repository. Add a `.gha-ignore` file with a list of files to be exluded. (Using glob patterns). |
 | `docker_repo_app_directory_cleanup` | Boolean | Will generate a timestamped compressed file (in the home directory of the instance) and delete the app repo directory. Runs before `docker_install` and after `docker_full_cleanup`. |
 | `docker_efs_mount_target` | String | Directory path within docker env to mount directory to. Default is `/data`|
+| `docker_cloudwatch_enable` | Boolean | Toggle cloudwatch creation for Docker. Create a file named `docker-daemon.json` in your repo root dir if you need to customize it. Defaults to `false`. Check [docker docs](https://docs.docker.com/config/containers/logging/awslogs/).|
+| `docker_cloudwatch_lg_name` | String| Log group name. Will default to `${aws_resource_identifier}-docker-logs` if none. |
+| `docker_cloudwatch_skip_destroy` | Boolean | Toggle deletion or not when destroying the stack. Defaults to `false`. |
+| `docker_cloudwatch_retention_days` | String | Number of days to retain logs. 0 to never expire. Defaults to `14`. See [note](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_log_group#retention_in_days). |
 <hr/>
 <br/>
 

action.yaml

@@ -414,6 +414,18 @@ inputs:
   docker_efs_mount_target:
     description: 'Directory path within docker env to mount directory to, default is `/data`'
     required: false
+  docker_cloudwatch_enable:
+    description: 'Toggle cloudwatch creation for Docker containers.'
+    required: false
+  docker_cloudwatch_lg_name:
+    description: 'Log group name. Will default to aws_identifier if none.'
+    required: false
+  docker_cloudwatch_skip_destroy:
+    description: 'Toggle deletion or not when destroying the stack.'
+    required: false
+  docker_cloudwatch_retention_days:
+    description: 'Number of days to retain logs. 0 to never expire.'
+    required: false
 
   # AWS ECR
   aws_ecr_repo_create:
@@ -736,6 +748,10 @@ runs:
         DOCKER_REPO_APP_DIRECTORY: ${{ inputs.docker_repo_app_directory }}
         DOCKER_REPO_APP_DIRECTORY_CLEANUP: ${{ inputs.docker_repo_app_directory_cleanup }}
         DOCKER_EFS_MOUNT_TARGET: ${{ inputs.docker_efs_mount_target }}
+        DOCKER_CLOUDWATCH_ENABLE: ${{ inputs.docker_cloudwatch_enable }}
+        DOCKER_CLOUDWATCH_LG_NAME: ${{ inputs.docker_cloudwatch_lg_name }}
+        DOCKER_CLOUDWATCH_SKIP_DESTROY: ${{ inputs.docker_cloudwatch_skip_destroy }}
+        DOCKER_CLOUDWATCH_RETENTION_DAYS: ${{ inputs.docker_cloudwatch_retention_days }}
 
         # ECR
         AWS_ECR_REPO_CREATE: ${{ inputs.aws_ecr_repo_create }}

operations/_scripts/generate/generate_bitops_config.sh

@@ -174,6 +174,10 @@ bitops:
     if [[ $(alpha_only "$AWS_EFS_CREATE") == true ]] || [[ $(alpha_only "$AWS_EFS_CREATE_HA") == true ]] || [[ "$AWS_EFS_FS_ID" != "" ]]; then
       add_ansible_module efs
     fi
+    # Ansible - Install CloudWatch
+    if [[ $(alpha_only "$DOCKER_CLOUDWATCH_ENABLE") == true ]]; then
+      add_ansible_module cloudwatch
+    fi
     # Ansible - Install Docker
     if [[ $(alpha_only "$DOCKER_INSTALL") == true ]]; then
       add_ansible_module docker

operations/_scripts/generate/generate_vars_terraform.sh

@@ -268,6 +268,10 @@ if [[ $(alpha_only "$DOCKER_INSTALL") == true ]]; then
   docker_install=$(generate_var docker_install $DOCKER_INSTALL)
   docker_remove_orphans=$(generate_var docker_remove_orphans $DOCKER_REMOVE_ORPHANS)
   docker_efs_mount_target=$(generate_var docker_efs_mount_target $DOCKER_EFS_MOUNT_TARGET)
+  docker_cloudwatch_enable=$(generate_var docker_cloudwatch_enable $DOCKER_CLOUDWATCH_ENABLE)
+  docker_cloudwatch_lg_name=$(generate_var docker_cloudwatch_lg_name $DOCKER_CLOUDWATCH_LG_NAME)
+  docker_cloudwatch_skip_destroy=$(generate_var docker_cloudwatch_skip_destroy $DOCKER_CLOUDWATCH_SKIP_DESTROY)
+  docker_cloudwatch_retention_days=$(generate_var docker_cloudwatch_retention_days $DOCKER_CLOUDWATCH_RETENTION_DAYS)
 fi
 
 #-- Application --#
@@ -445,6 +449,10 @@ $aws_eks_additional_tags
 
 $docker_efs_mount_target
 $docker_remove_orphans
+$docker_cloudwatch_enable
+$docker_cloudwatch_lg_name
+$docker_cloudwatch_skip_destroy
+$docker_cloudwatch_retention_days
 
 #-- Application --#
 $ops_repo_environment

operations/deployment/ansible/cloudwatch/playbook.yml

@@ -2,5 +2,5 @@
   hosts: bitops_servers
   become: true
   tasks:
-  - name: Include install docker
+  - name: Include install Cloudwatch
     include_tasks: tasks/install.yml

operations/deployment/ansible/cloudwatch/tasks/install.yml

@@ -1,12 +1,28 @@
-# copy code to the VM
-- name: Install the aws ca bundle
-  get_url:
-    url: "https://s3.amazonaws.com/amazoncloudwatch-agent/ubuntu/arm64/latest/amazon-cloudwatch-agent.deb"
-    dest: "{{ app_install_root }}/amazon-cloudwatch-agent.deb"
-
-- name: Install cloudwatch agent
-  apt:
-    deb: "{{ app_install_root }}/amazon-cloudwatch-agent.deb"
-  become: yes
-
-#  https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/install-CloudWatch-Agent-commandline-fleet.html
+- name: Check provided Docker Cloudwatch config
+  ansible.builtin.find:
+    paths: "{{ app_install_root }}/{{ app_repo_name }}"
+    recurse: no
+    patterns: 'docker-daemon.json'
+  register: file_status_d
+
+- name: Create the docker config folder
+  file:
+    path: /etc/docker
+    state: directory
+    mode: '0755'
+
+- name: Copy the Docker cloudwatch config if it doesn't exists
+  copy:
+    src: "{{ inventory_dir }}/bitovi-daemon.json"
+    dest: "/etc/docker/daemon.json"
+  when: file_status_d.matched == 0
+
+- name: Copy the Docker cloudwatch config if it does exists
+  copy:
+    src: "{{ app_install_root }}/{{ app_repo_name }}/docker-daemon.json"
+    dest: "/etc/docker/daemon.json"
+  when: file_status_d.matched == 1
+
+- name: Restart docker service for changes to take effect
+  command: systemctl restart docker
+  become: yes

operations/deployment/ansible/cloudwatch/tasks/install_agent.yml

@@ -0,0 +1,29 @@
+- name: Get APT package architecture
+  command: dpkg --print-architecture
+  register: dpkg_arch
+
+- name: Download the aws Cloudwatch agent
+  get_url:
+    url: "https://amazoncloudwatch-agent.s3.amazonaws.com/ubuntu/{{ dpkg_arch.stdout }}/latest/amazon-cloudwatch-agent.deb"
+    dest: "{{ app_install_root }}/amazon-cloudwatch-agent.deb"
+
+- name: Install the aws Cloudwatch agent
+  apt:
+    deb: "{{ app_install_root }}/amazon-cloudwatch-agent.deb"
+
+- name: Move provided Cloudwatch config
+  ansible.builtin.find:
+    paths: "{{ app_install_root }}/{{ app_repo_name }}"
+    recurse: no
+    patterns: 'cloudwatch.json'
+  register: file_status
+
+- name: Copy the cloudwatch config if it doesn't exists
+  copy:
+    src: "{{ inventory_dir }}/bitovi-cloudwatch.json"
+    dest: "{{ app_install_root }}/{{ app_repo_name }}/cloudwatch.json"
+  when: file_status.matched == 0
+
+- name: Trigger Cloudwatch start with the config
+  ansible.builtin.shell: >
+    /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -c "file:{{ app_install_root }}/{{ app_repo_name }}/cloudwatch.json" -s

operations/deployment/ansible/docker/tasks/install.yml

@@ -14,26 +14,20 @@
     path: /etc/apt/keyrings/docker.asc
     mode: '0666'
 
+- name: Get APT package architecture
+  command: dpkg --print-architecture
+  register: dpkg_arch
+
 - name: Add Docker repo
   apt_repository:
-    repo: "deb [arch=amd64 signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu {{ ansible_distribution_release }} stable"
+    repo: "deb [arch={{ dpkg_arch.stdout }} signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu {{ ansible_distribution_release }} stable"
     state: present
 
 - name: Update apt repositories
   apt:
     update_cache: yes
-  
+
 - name: Install Docker and Docker Compose
   apt:
     name:
-      - docker-ce
-      - docker-ce-cli
-      - containerd.io
-      - docker-buildx-plugin
-      - docker-compose-plugin
-
-- name: Install Docker Module for Python
-  pip:
-    name:
-      - docker
-      - docker-compose
+      - docker-ce

operations/deployment/ansible/docker/tasks/start.yml

@@ -17,12 +17,19 @@
     msg: "Can't find a valid compose file. Expected one of: docker-compose.yml, docker-compose.yaml, compose.yml, compose.yaml. Not running docker-compose up."
   when: existing_files | length < 1
 
-- name: Start docker-compose
-  ansible.builtin.command: "docker compose --project-directory {{ app_install_root }}/{{ app_repo_name }} up --detach --build --force-recreate --remove-orphans={{ docker_remove_orphans }}"
+- name: Start docker-compose with remove orphans
+  ansible.builtin.command: "docker compose --project-directory {{ app_install_root }}/{{ app_repo_name }} up --detach --build --force-recreate --remove-orphans"
   register: output
   async: "{{ ansible_start_docker_timeout }}"
   poll: 5
-  when: existing_files | length > 0
+  when: existing_files | length > 0 and ( docker_remove_orphans | bool )
+
+- name: Start docker-compose without remove orphans
+  ansible.builtin.command: "docker compose --project-directory {{ app_install_root }}/{{ app_repo_name }} up --detach --build --force-recreate"
+  register: output
+  async: "{{ ansible_start_docker_timeout }}"
+  poll: 5
+  when: existing_files | length > 0 and not ( docker_remove_orphans | bool )
 
 - ansible.builtin.debug:
     var: output

operations/deployment/ansible/docker_cleanup/tasks/docker_cleanup.yml

@@ -31,4 +31,8 @@
 
 - name: Prune Docker system
   command: docker system prune --all --force --volumes
+  when: docker_check.rc == 0
+
+- name: Prune all docker volumes
+  command: docker volume prune --all --force
   when: docker_check.rc == 0

operations/deployment/terraform/aws/aws_variables.tf

@@ -134,6 +134,7 @@ variable "aws_ec2_additional_tags" {
   default     = "{}"
 }
 
+
 ## AWS VPC
 variable "aws_vpc_create" {
   type        = bool
@@ -559,6 +560,30 @@ variable "docker_remove_orphans" {
   default     = false
 }
 
+variable "docker_cloudwatch_enable" {
+  type        = bool
+  description = "Toggle cloudwatch for Docker."
+  default     = false
+}
+
+variable "docker_cloudwatch_lg_name" {
+  type        = string
+  description = "Log group name. Will default to aws_identifier if none."
+  default     = ""
+}
+
+variable "docker_cloudwatch_skip_destroy" {
+  type        = bool
+  description = "Toggle deletion or not when destroying the stack."
+  default     = false
+}
+
+variable "docker_cloudwatch_retention_days" {
+  type        = string
+  description = "Number of days to retain logs. 0 to never expire."
+  default     = "14"
+}
+
 # ECR
 variable "aws_ecr_repo_create" { 
   description = "Determines whether a repository will be created"

operations/deployment/terraform/aws/bitovi_main.tf

@@ -286,19 +286,25 @@ module "secretmanager_get" {
 module "ansible" {
   source = "../modules/aws/ansible"
   count  = var.ansible_skip ? 0 : var.aws_ec2_instance_create ? 1 : 0
-  aws_ec2_instance_ip          = var.ansible_ssh_to_private_ip ? module.ec2[0].instance_private_ip : ( module.ec2[0].instance_public_ip != "" ? module.ec2[0].instance_public_ip : module.ec2[0].instance_private_ip )
-  ansible_start_docker_timeout = var.ansible_start_docker_timeout
-  aws_efs_enable               = var.aws_efs_enable
-  app_repo_name                = var.app_repo_name
-  app_install_root             = var.app_install_root
-  aws_resource_identifier      = var.aws_resource_identifier
-  docker_remove_orphans        = var.docker_remove_orphans
-  aws_efs_ec2_mount_point      = var.aws_efs_ec2_mount_point
-  aws_efs_mount_target         = var.aws_efs_mount_target
-  docker_efs_mount_target      = var.docker_efs_mount_target
-  aws_efs_fs_id                = var.aws_efs_enable ? local.create_efs ? module.efs[0].aws_efs_fs_id : var.aws_efs_fs_id : null
+  aws_ec2_instance_ip              = var.ansible_ssh_to_private_ip ? module.ec2[0].instance_private_ip : ( module.ec2[0].instance_public_ip != "" ? module.ec2[0].instance_public_ip : module.ec2[0].instance_private_ip )
+  ansible_start_docker_timeout     = var.ansible_start_docker_timeout
+  aws_efs_enable                   = var.aws_efs_enable
+  app_repo_name                    = var.app_repo_name
+  app_install_root                 = var.app_install_root
+  aws_resource_identifier          = var.aws_resource_identifier
+  docker_remove_orphans            = var.docker_remove_orphans
+  # Cloudwatch
+  docker_cloudwatch_enable         = var.docker_cloudwatch_enable
+  docker_cloudwatch_lg_name        = var.docker_cloudwatch_lg_name != "" ? var.docker_cloudwatch_lg_name : "${var.aws_resource_identifier}-docker-logs"
+  docker_cloudwatch_skip_destroy   = var.docker_cloudwatch_skip_destroy
+  docker_cloudwatch_retention_days = var.docker_cloudwatch_retention_days
+  aws_region_current_name          = module.vpc.aws_region_current_name
+  aws_efs_ec2_mount_point          = var.aws_efs_ec2_mount_point
+  aws_efs_mount_target             = var.aws_efs_mount_target
+  docker_efs_mount_target          = var.docker_efs_mount_target
+  aws_efs_fs_id                    = var.aws_efs_enable ? local.create_efs ? module.efs[0].aws_efs_fs_id : var.aws_efs_fs_id : null
   # Data inputs
-  private_key_filename         = module.ec2[0].private_key_filename
+  private_key_filename             = module.ec2[0].private_key_filename
   # Dependencies
   depends_on = [module.ec2]
 }

operations/deployment/terraform/modules/aws/ansible/aws_ansible_cloudwatch.tf

@@ -0,0 +1,51 @@
+
+resource "aws_cloudwatch_log_group" "cw_log_group" {
+  count             = var.docker_cloudwatch_enable ? 1 : 0
+  name              = var.docker_cloudwatch_lg_name
+  skip_destroy      = var.docker_cloudwatch_skip_destroy
+  retention_in_days = tonumber(var.docker_cloudwatch_retention_days)
+}
+
+resource "local_file" "cloudwatch_docker_config" {
+  count = var.docker_cloudwatch_enable ? 1 : 0
+  filename = format("%s/%s", abspath(path.root), "bitovi-daemon.json")
+  content = <<-EOT
+{
+  "log-driver": "awslogs",
+  "log-opts": {
+    "awslogs-region": "${var.aws_region_current_name}",
+    "awslogs-group": "${var.docker_cloudwatch_lg_name}",
+    "tag": "{{.Name}}"
+  }
+}
+EOT
+}
+
+#resource "local_file" "cloudwatch_config" {
+#  count = var.aws_ec2_cloudwatch_enable ? 1 : 0
+#  filename = format("%s/%s", abspath(path.root), "bitovi-cloudwatch.json")
+#  content = <<-EOT
+#{
+#    "agent": {
+#      "metrics_collection_interval": 5,
+#      "run_as_user": "root"
+#    },
+#    "logs": {
+#      "logs_collected": {
+#            "files": {
+#              "collect_list": [
+#                    {
+#                      "file_path": "/var/lib/docker/containers/*/*.log",
+#                      "log_group_name": "${var.aws_ec2_cloudwatch_lg_name}",
+#                      "log_stream_name": "{instance_id}",
+#                      "timestamp_format": "%b %d %H:%M:%S",
+#                      "timezone": "Local"
+#                    }
+#              ]
+#            }
+#      }
+#    }
+#}
+#EOT
+#}
+

operations/deployment/terraform/modules/aws/ansible/aws_ansible_inventory_vars.tf

@@ -5,6 +5,11 @@ variable "app_repo_name" {}
 variable "app_install_root" {}
 variable "aws_resource_identifier" {}
 variable "docker_remove_orphans" {}
+variable "docker_cloudwatch_enable" {}
+variable "docker_cloudwatch_lg_name" {}
+variable "docker_cloudwatch_skip_destroy" {}
+variable "docker_cloudwatch_retention_days" {}
+variable "aws_region_current_name" {}
 variable "aws_efs_ec2_mount_point" {}
 variable "aws_efs_mount_target" {}
 variable "docker_efs_mount_target" {}