Merge pull request #30 from EngineersBox/UIUX-74-New-Box-Ansible-GH-W…

…orkflow-Updates

.github/listContains.sh

@@ -6,5 +6,5 @@ printExit() {
 }
 
 for value in $2; do
-    [[ $1 =~ (^|[[:space:]])$value($|[[:space:]]) ]] || printExit "Invlid input: $value"
+    [[ $1 =~ (^|[[:space:]])$value($|[[:space:]]) ]] || printExit "Invalid input: $value"
 done

.github/workflows/ansible.yml

@@ -11,6 +11,10 @@ on:
         description: "Level of verbosity to run Absible at. Must be one of: [0, 1, 2, 3]"
         required: true
         default: '1'
+      target:
+        description: "Box to target deployment to. Must be one of: [old, prod]"
+        required: true
+        default: 'old'
 
 jobs:
   build:
@@ -31,8 +35,12 @@ jobs:
       - name: Create verbosity level flags
         run: |
           verbosity_tags="$(echo "-$(printf "%3s" | tr " " "v")")"
-          echo "verbosity_tags=\"$verbosity_tags\"" >> $GITHUB_ENV
+          echo "verbosity_tags=$verbosity_tags" >> $GITHUB_ENV
           echo "Generated verbosity flag: [$verbosity_tags]"
+      - name: Mapping target to inventory
+        run: |
+          echo "target_inventory=inventory/${{ github.event.inputs.target }}.ini" >> $GITHUB_ENV
+          echo "Added specifier for target inventory"
       - name: Install ansible
         run: |
           sudo apt update
@@ -46,13 +54,15 @@ jobs:
           sudo chmod 0600 $PRIVATE_KEY_FILE
         working-directory: ${{ env.ANSIBLE_DIR }}
         env:
-          SSH_PRIVATE_KEY: ${{ secrets.DEPLOY_KEY }}
+          SSH_PRIVATE_KEY: if [[ "${{ github.event.inputs.target }}" == "old" ]]; then echo ${{ secrets.OLD_DEPLOY_KEY }}; else if [[ "${{ github.event.inputs.target }}" == "prod" ]]; then echo ${{ secrets.PROD_DEPLOY_KEY }}; else echo "INVALID"; fi;
       - name: Verify targets are accessible
-        run: ansible all --private-key=$PRIVATE_KEY_FILE -m ping
+        run: ansible all ${{ env.verbosity_tags }} --ssh-extra-args=${{ env.verbosity_tags }} -i ${{ env.target_inventory }} --private-key=$PRIVATE_KEY_FILE -m ping
         working-directory: ${{ env.ANSIBLE_DIR }}
       - name: Run playbook
-        run: ansible-playbook ${{ env.verbosity_tags }} --tags "$(echo "${{ github.event.inputs.tags }}" | tr -ds "[:space:]" "")"  --private-key=$PRIVATE_KEY_FILE --extra-vars "ansible_sudo_pass=${{ secrets.SUDO_PASS }}" $PLAYBOOK
+        run: ansible-playbook ${{ env.verbosity_tags }} --ssh-extra-args=${{ env.verbosity_tags }}  -i ${{ env.target_inventory }} --tags "$(echo "${{ github.event.inputs.tags }}" | tr -ds "[:space:]" "")"  --private-key=$PRIVATE_KEY_FILE --extra-vars "ansible_sudo_pass=${{ env.SSH_USER_SUDO_PASS }}" $PLAYBOOK
         working-directory: ${{ env.ANSIBLE_DIR }}
+        env:
+          SSH_USER_SUDO_PASS: if [[ "${{ github.event.inputs.target }}" == "old" ]]; then echo ${{ secrets.OLD_SUDO_PASS }}; else if [[ "${{ github.event.inputs.target }}" == "prod" ]]; then echo ${{ secrets.PROD_SUDO_PASS }}; else echo "INVALID"; fi;
       - name: Remove key file
         run: sudo rm -rf $PRIVATE_KEY_FILE
         working-directory: ${{ env.ANSIBLE_DIR }}

Dockerfile

@@ -4,10 +4,19 @@ ENV PYTHONBUFFERED 1
 RUN mkdir -p /app
 WORKDIR /app
 
-COPY . .
+COPY app.py app.py
+COPY fileLogHandler.py fileLogHandler.py
+COPY flask_logging.conf flask_logging.conf
+COPY requirements.txt requirements.txt
+COPY minitask minitask
+COPY knn_indexing knn_indexing
+COPY summary summary
+COPY summary_1 summary_1
+COPY modules/RateLimiter modules/RateLimiter
 
 RUN python3 -m pip install --upgrade pip
 RUN python3 -m pip install --no-cache-dir -r requirements.txt
 RUN python3 -m spacy download en_core_web_trf
 
 EXPOSE 3001
+CMD ["gunicorn", "-w", "4", "-b", "0.0.0.0:3001", "app:app"]

ansible/ansible.cfg

@@ -1,3 +1,2 @@
 [defaults]
 host_key_checking=False
-inventory=inventories/inventory.ini

ansible/inventory/prod.ini

@@ -0,0 +1,2 @@
+[targets]
+ansible@65.21.139.236

ansible/roles/backend/tasks/main.yml

@@ -14,7 +14,7 @@
 
 - name: Spin up new containers
   docker_compose:
-    project_src: "temp"
+    project_src: "temp/compose/backend"
     build: yes
     debug: yes
   become: true

ansible/roles/frontend/tasks/main.yml

@@ -14,7 +14,7 @@
 
 - name: Spin up new containers
   docker_compose:
-    project_src: "temp/frontend"
+    project_src: "temp/compose/frontend"
     build: yes
     debug: yes
   become: true

ansible/roles/elasticsearch/tasks/main.yml → ansible/roles/opensearch/tasks/main.yml

@@ -1,46 +1,51 @@
 - name: Stop and remove container
   vars:
     container_names:
-      - elasticsearch
+      - opensearch-master
+      - opensearch-data-ingest1
+      - opensearch-data-ingest2
+      - opensearch-data-ingest3
+      - opensearch-coordinator
+      - opensearch-dashboards
     containers: "{{ item }}"
   include_tasks:
     file: roles/common/tasks/docker_stop_remove_tasks.yml
     apply:
       tags:
-        - elasticsearch
+        - opensearch
   loop: "{{ container_names | batch(1) | list }}"
   tags:
-    - elasticsearch
+    - opensearch
 
 - name: Spin up new container
   docker_compose:
-    project_src: "temp/compose/elasticsearch"
+    project_src: "temp/compose/opensearch"
     build: true
     debug: true
   become: true
   register: started_containers
   tags:
-    - elasticsearch
+    - opensearch
 
-- name: Wait for elasticsearch to start
-  command: docker exec elasticsearch curl localhost:9200/_recovery -sS
+- name: Wait for opensearch to start
+  command: docker exec opensearch curl localhost:9200/_recovery -sS
   register: result
   until: result.stdout == '{}'
   retries: 30
   delay: 5 # Seconds
   become: true
   tags:
-    - elasticsearch
-    - elasticsearch_base_indexes
-    - elasticsearch_knn_indexes
+    - opensearch
+    - opensearch_base_indexes
+    - opensearch_knn_indexes
 
 - name: Create base indexes
   shell:
     cmd: "python3.8 import_data.py"
     chdir: "temp/minitask"
   register: create_base
   tags:
-    - elasticsearch_base_indexes
+    - opensearch_base_indexes
 
 - debug: var=create_base
 
@@ -50,6 +55,6 @@
     chdir: "temp/knn_indexing"
   register: create_knn
   tags:
-    - elasticsearch_knn_indexes
+    - opensearch_knn_indexes
 
 - debug: var=create_knn

ansible/site.yml

@@ -17,7 +17,7 @@
 
 - hosts: targets
   roles:
-    - elasticsearch
+    - opensearch
 
 - hosts: targets
   roles:

caddy/Caddyfile

@@ -0,0 +1,7 @@
+anu.jkl.io {
+	reverse_proxy localhost:3002
+	route /api/* {
+		uri strip_prefix /api
+		reverse_proxy localhost:3001
+	}
+}

caddy/Dockerfile

@@ -0,0 +1,6 @@
+FROM caddy:2-alpine
+
+COPY Caddyfile /etc/caddy/Caddyfile
+
+EXPOSE 80
+EXPOSE 443

compose/backend/docker-compose.yml

@@ -0,0 +1,7 @@
+version: "3.5"
+services:
+  backend:
+    container_name: backend
+    network_mode: host
+    build:
+      context: ../../

compose/caddy/docker-compose.yml

@@ -0,0 +1,8 @@
+version: "3.5"
+services:
+  caddy:
+    container_name: caddy
+    network_mode: host
+    build:
+      context: ../../caddy
+    restart: unless-stopped

compose/frontend/docker-compose.yml

@@ -0,0 +1,6 @@
+version: "3.5"
+services:
+  new-frontend:
+    build:
+      context: ../../frontend_new
+    network_mode: host

compose/opensearch/docker-compose.yml

@@ -23,9 +23,7 @@ services:
                 hard: 65536
         volumes:
             - opensearch-master:/usr/share/opensearch/data
-        ports:
-            - 9200:9200
-            - 9600:9600 # required for Performance Analyzer
+        network_mode: host
         networks:
             - opensearch-net
     opensearch-data-ingest1:
@@ -49,9 +47,7 @@ services:
             nofile:
                 soft: 65536
                 hard: 65536
-        ports:
-            - 9201:9201
-            - 9601:9601 # required for Performance Analyzer
+        network_mode: host
         volumes:
             - opensearch-data-ingest1:/usr/share/opensearch/data
         networks:
@@ -77,9 +73,7 @@ services:
             nofile:
                 soft: 65536
                 hard: 65536
-        ports:
-            - 9202:9202
-            - 9602:9602 # required for Performance Analyzer
+        network_mode: host
         volumes:
             - opensearch-data-ingest2:/usr/share/opensearch/data
         networks:
@@ -105,9 +99,7 @@ services:
             nofile:
                 soft: 65536
                 hard: 65536
-        ports:
-            - 9203:9203
-            - 9603:9603 # required for Performance Analyzer
+        network_mode: host
         volumes:
             - opensearch-data-ingest2:/usr/share/opensearch/data
         networks:
@@ -132,20 +124,15 @@ services:
             nofile:
                 soft: 65536
                 hard: 65536
-        ports:
-            - 9204:9204
-            - 9604:9604 # required for Performance Analyzer
+        network_mode: host
         volumes:
             - opensearch-coordinator:/usr/share/opensearch/data
         networks:
             - opensearch-net
     opensearch-dashboards:
         image: opensearchproject/opensearch-dashboards:latest
         container_name: opensearch-dashboards
-        ports:
-            - 5601:5601
-        expose:
-            - "5601"
+        network_mode: host
         environment:
             OPENSEARCH_HOSTS: '["http://opensearch-master:9200","http://opensearch-master:9201","http://opensearch-master:9202","http://opensearch-master:9203","http://opensearch-master:9204"]'
         networks:

compose/opensearch/opensearch.yml

@@ -7,12 +7,6 @@ network.host: 0.0.0.0
 # # set to 1 to allow single node clusters
 # discovery.zen.minimum_master_nodes: 1
 
-node.name: "node-1"
-# explicitly specify the node will be function as master, data and ingest
-node.master: true
-node.data: true
-node.ingest: true
-
 # Setting network.host to a non-loopback address enables the annoying bootstrap checks. "Single-node" mode disables them again.
 #discovery.type: single-node
 

compose/redis/docker-compose.yml

@@ -7,4 +7,3 @@ services:
             context: ../../modules/CuckooFilter
         volumes:
             - ../redis:/redis
-        command: redis-server /usr/local/etc/redis/redis.conf

frontend_new/Dockerfile

@@ -0,0 +1,12 @@
+FROM node:lts-alpine
+
+RUN npm install -g http-server
+
+WORKDIR /app
+
+COPY . .
+RUN npm install
+RUN npm run build
+
+EXPOSE 3002
+CMD ["http-server", "-p", "3002"]