Merge pull request #30 from vectornguyen76/staging

Develop CICD

.github/workflows/README.md

@@ -1,46 +1,65 @@
-# Workflows
+# CI/CD Deploy
 
-## Development - Build and Unittest
+## Workflows
 
-File: [development_pipeline.yml](development_pipeline.yml)
+### Development - Build and Unittest
 
-Event: On **Pull Request** → any branch into **develop**
+#### File: [development_pipeline.yml](development_pipeline.yml)
 
-Jobs:
+**Event:** On Pull Request → any branch into develop
+
+**Jobs:**
 
 - Install dependencies (caches)
 - Run isort
 - Run black
 - Run flake8
 - Build images (caches)
-- Push images to docker hub
-
-### Description:
+- Push images to Docker Hub
 
-## Staging - CI/CD Pipeline
+**Description:**
+This workflow is triggered on Pull Requests into the develop branch. It ensures a clean and standardized codebase by installing dependencies, checking code formatting with isort, black, and flake8, and finally building and pushing Docker images to Docker Hub.
 
-File: [staging_pipeline.yml](staging_pipeline.yml)
+### Staging - CI/CD Pipeline
 
-Event: On **Pull Request** → any branch into **staging**
+#### File: [staging_pipeline.yml](staging_pipeline.yml)
 
-Jobs:
+**Event:** On Pull Request → any branch into staging
 
-- Build
-- Unit Test
-- Deploy
+**Jobs:**
 
-### Description:
+- Install dependencies (caches)
+- Run isort
+- Run black
+- Run flake8
+- Build images (caches)
+- Push images to Docker Hub
+- Create infrastructure
+- Configure infrastructure
+- Deploy application using Docker Compose
+- Clean up following the concept of A/B deploy
 
-## Production - CI/CD Pipeline
+**Description:**
+This pipeline is designed for the staging environment and is triggered on Pull Requests into the staging branch. It includes steps to ensure code quality, build and push Docker images, create and configure necessary infrastructure, and deploy the application using Docker Compose. The cleanup process follows the A/B deployment concept.
 
-File: [production_pipeline.yml](production_pipeline.yml)
+### Production - CI/CD Pipeline
 
-Event: On **Pull Request** → any branch into **master**
+#### File: [production_pipeline.yml](production_pipeline.yml)
 
-Jobs:
+**Event:** On Pull Request → any branch into master
 
-- Build
-- Test
-- Deploy
+**Jobs:**
 
-### Description:
+- Install dependencies (caches)
+- Run isort
+- Run black
+- Run flake8
+- Build images (caches)
+- Push images to Docker Hub
+- Create infrastructure
+- Configure infrastructure
+- Deploy application using Docker Compose
+- Clean up following the concept of A/B deploy
+
+**Description:**
+The production pipeline is triggered on Pull Requests into the master branch, indicating changes are ready for deployment to the production environment. It follows a similar process to the staging pipeline but is specifically tailored for the production environment. The cleanup process adheres to the A/B deployment concept, ensuring a smooth transition between versions.

.github/workflows/ansible/deploy-app.yml → .github/workflows/ansible/deploy_applications.yml

@@ -1,5 +1,5 @@
 ---
-- name: "configuration play."
+- name: "Deploy applications"
   hosts: web
   user: ubuntu
   become: true
@@ -9,11 +9,18 @@
     - ansible_host_key_checking: false
     - ansible_stdout_callback: yaml
 
-    - mlflow_image_name: vectornguyen76/mlflow
-    - mlflow_tag_name: latest
+    - image_search_image: vectornguyen76/image-search-engine
+    - image_search_tag: latest
+
+    - text_search_image: vectornguyen76/text-search-engine
+    - text_search_tag: latest
+
+    - backend_image: vectornguyen76/backend-search-engine
+    - backend_tag: latest
+
+    - frontend_image: vectornguyen76/frontend-search-engine
+    - frontend_tag: latest
 
-    - model_predictor_image_name: vectornguyen76/model_predictor
-    - model_predictor_tag_name: latest
   pre_tasks:
     - name: "wait 600 seconds for target connection to become reachable/usable."
       wait_for_connection:
@@ -34,6 +41,7 @@
           - python3-pip
           - virtualenv
           - python3-setuptools
+          - unzip
         state: latest
         update_cache: true
 

.github/workflows/ansible/roles/deploy/tasks/main.yml

@@ -6,35 +6,41 @@
 
 - name: "Copy compressed app folder"
   copy:
-    src: "artifact-app.tar.gz"
-    dest: "/home/ubuntu/server/artifact-app.tar.gz"
+    src: "artifact.zip"
+    dest: "/home/ubuntu/server/artifact.zip"
 
 - name: "Extract app"
-  unarchive:
+  ansible.builtin.unarchive:
     remote_src: yes
-    src: "/home/ubuntu/server/artifact-app.tar.gz"
+    src: "/home/ubuntu/server/artifact.zip"
     dest: "/home/ubuntu/server"
 
-- name: Pull mlflow image
+- name: Pull image search image
   community.docker.docker_image:
-    name: "{{ mlflow_image_name }}"
-    tag: "{{ mlflow_tag_name }}"
+    name: "{{ image_search_image }}"
+    tag: "{{ image_search_tag }}"
     source: pull
 
-- name: Pull model_predictor image
+- name: Pull text search image
   community.docker.docker_image:
-    name: "{{ model_predictor_image_name }}"
-    tag: "{{ model_predictor_tag_name }}"
+    name: "{{ text_search_image }}"
+    tag: "{{ text_search_tag }}"
     source: pull
 
-- name: Run mlflow container
-  become: True
-  shell:
-    chdir: /home/ubuntu/server
-    cmd: "docker compose -f deployment/mlflow/docker-compose.yml up -d"
+- name: Pull backend image
+  community.docker.docker_image:
+    name: "{{ backend_image }}"
+    tag: "{{ backend_tag }}"
+    source: pull
+
+- name: Pull frontend image
+  community.docker.docker_image:
+    name: "{{ frontend_image }}"
+    tag: "{{ frontend_tag }}"
+    source: pull
 
-- name: Run model_predictor container
+- name: Run docker compose
   become: True
   shell:
     chdir: /home/ubuntu/server
-    cmd: "bash deployment/deploy.sh deploy_run_predictor data/model_config/phase-1/prob-1/model-1.yml data/model_config/phase-1/prob-2/model-1.yml 5040"
+    cmd: "docker compose --profile dev up -d"

.github/workflows/cloudformations/README.md

@@ -0,0 +1,7 @@
+## Create Stack
+
+aws cloudformation create-stack --stack-name server --template-body file://server.yml --parameters file://server-parameters.json --region us-east-1
+
+## Delete Stack
+
+aws cloudformation delete-stack --stack-name server --region us-east-1

.github/workflows/files/ec2-parameters.json → .github/workflows/cloudformations/server-parameters.json

@@ -1,4 +1,8 @@
 [
+  {
+    "ParameterKey": "EnvironmentName",
+    "ParameterValue": "Search-Engine"
+  },
   {
     "ParameterKey": "VpcCIDR",
     "ParameterValue": "10.0.0.0/16"
@@ -12,7 +16,7 @@
     "ParameterValue": "t3.medium"
   },
   {
-    "ParameterKey": "KeyPair",
+    "ParameterKey": "KeyPairName",
     "ParameterValue": "my-keypair"
   },
   {

.github/workflows/files/ec2.yml → .github/workflows/cloudformations/server.yml

@@ -1,6 +1,10 @@
 AWSTemplateFormatVersion: 2010-09-09
-Description: Creates EC2
+Description: Creates EC2 Server
 Parameters:
+  EnvironmentName:
+    Description: An environment name that will be prefixed to resource names
+    Type: String
+
   VpcCIDR:
     Description: "VPC range"
     Type: String
@@ -38,7 +42,7 @@ Resources:
       EnableDnsSupport: true
       Tags:
         - Key: Name
-          Value: "Main VPC"
+          Value: !Sub ${EnvironmentName} VPC
 
   InternetGateway:
     Type: AWS::EC2::InternetGateway
@@ -56,12 +60,12 @@ Resources:
   PublicSubnet:
     Type: AWS::EC2::Subnet
     Properties:
-      AvailabilityZone: "us-east-1a"
+      AvailabilityZone: !Select [0, !GetAZs ""]
       VpcId: !Ref VPC
       CidrBlock: !Ref PublicSubnetCIDR
       Tags:
         - Key: Name
-          Value: "Public Subnet AZ1"
+          Value: !Sub ${EnvironmentName} Public Subnet
 
   PublicRouteTable:
     Type: AWS::EC2::RouteTable
@@ -88,14 +92,18 @@ Resources:
   SecurityGroup:
     Type: AWS::EC2::SecurityGroup
     Properties:
-      GroupName: mySecurityGroup
+      GroupName: !Sub ${EnvironmentName}-SecurityGroup
       GroupDescription: Allow http to client host
       VpcId: !Ref VPC
       SecurityGroupIngress:
         - IpProtocol: tcp
           FromPort: 80
           ToPort: 80
           CidrIp: 0.0.0.0/0
+        - IpProtocol: tcp
+          FromPort: 443
+          ToPort: 443
+          CidrIp: 0.0.0.0/0
         - IpProtocol: tcp
           FromPort: 22
           ToPort: 22
@@ -108,18 +116,11 @@ Resources:
           FromPort: 3000
           ToPort: 3000
           CidrIp: 0.0.0.0/0
-        - IpProtocol: tcp
-          FromPort: 5040
-          ToPort: 5040
-          CidrIp: 0.0.0.0/0
       SecurityGroupEgress:
         - IpProtocol: "-1"
           FromPort: -1
           ToPort: -1
           CidrIp: 0.0.0.0/0
-      Tags:
-        - Key: Name
-          Value: SecurityGroup
 
   # Server EC2 Instance
   ServerInstance:
@@ -135,27 +136,12 @@ Resources:
             - !Ref SecurityGroup
           SubnetId: !Ref PublicSubnet
       BlockDeviceMappings:
-        - DeviceName: "/dev/sdk"
+        - DeviceName: "/dev/sda1"
           Ebs:
-            VolumeSize: 20
-      # UserData:
-      #   Fn::Base64: !Sub |
-      #     #!/bin/bash
-      #     sudo apt-get update -y
-      #     sudo apt-get install ca-certificates curl gnupg
-      #     sudo install -m 0755 -d /etc/apt/keyrings
-      #     curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg
-      #     sudo chmod a+r /etc/apt/keyrings/docker.gpg
-      #     echo \
-      #     "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \
-      #     "$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" | \
-      #     sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
-      #     sudo apt-get update -y
-      #     apt-get install -y docker-ce
-      #     docker run -p 80:8080 tomcat:8.0
+            VolumeSize: 50
       Tags:
         - Key: Name
-          Value: ServerInstance
+          Value: !Sub ${EnvironmentName}-Instance
 
 Outputs:
   EC2InstanceConnection:

.github/workflows/development_pipeline.yml

@@ -1,17 +1,12 @@
 name: development
 on:
   pull_request:
-    branches: develop
-
-  push:
     branches:
       - develop
+
 jobs:
   build-push-image-search:
     runs-on: ubuntu-latest
-    env:
-      DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
-      DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
     steps:
       - name: Checkout Repository
         uses: actions/checkout@v2
@@ -65,9 +60,6 @@ jobs:
 
   build-push-text-search:
     runs-on: ubuntu-latest
-    env:
-      DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
-      DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
     steps:
       - name: Checkout Repository
         uses: actions/checkout@v2
@@ -121,9 +113,6 @@ jobs:
 
   build-push-backend:
     runs-on: ubuntu-latest
-    env:
-      DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
-      DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
     steps:
       - name: Checkout Repository
         uses: actions/checkout@v2
@@ -177,12 +166,6 @@ jobs:
 
   build-push-frontend:
     runs-on: ubuntu-latest
-    env:
-      DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
-      DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
-      GOOGLE_CLIENT_ID: ${{ secrets.GOOGLE_CLIENT_ID }}
-      GOOGLE_CLIENT_SECRET: ${{ secrets.GOOGLE_CLIENT_SECRET }}
-      NEXTAUTH_SECRET: ${{ secrets.NEXTAUTH_SECRET }}
     steps:
       - name: Checkout Repository
         uses: actions/checkout@v2
@@ -199,9 +182,9 @@ jobs:
 
       - name: Add env variable to env file
         run: |
-          echo GOOGLE_CLIENT_ID=${{ env.GOOGLE_CLIENT_ID }} >> ./frontend/.env
-          echo GOOGLE_CLIENT_SECRET=${{ env.GOOGLE_CLIENT_SECRET }} >> ./frontend/.env
-          echo NEXTAUTH_SECRET=${{ env.NEXTAUTH_SECRET }} >> ./frontend/.env
+          echo GOOGLE_CLIENT_ID=${{ secrets.GOOGLE_CLIENT_ID }} >> ./frontend/.env
+          echo GOOGLE_CLIENT_SECRET=${{ secrets.GOOGLE_CLIENT_SECRET }} >> ./frontend/.env
+          echo NEXTAUTH_SECRET=${{ secrets.NEXTAUTH_SECRET }} >> ./frontend/.env
 
       - name: Build and push frontend image
         uses: docker/build-push-action@v4