Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

UIUX-74 New box deployment for caddy and frontend #30

Merged
merged 18 commits into from
Oct 6, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/listContains.sh
Original file line number Diff line number Diff line change
Expand Up @@ -6,5 +6,5 @@ printExit() {
}

for value in $2; do
[[ $1 =~ (^|[[:space:]])$value($|[[:space:]]) ]] || printExit "Invlid input: $value"
[[ $1 =~ (^|[[:space:]])$value($|[[:space:]]) ]] || printExit "Invalid input: $value"
done
18 changes: 14 additions & 4 deletions .github/workflows/ansible.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,10 @@ on:
description: "Level of verbosity to run Absible at. Must be one of: [0, 1, 2, 3]"
required: true
default: '1'
target:
description: "Box to target deployment to. Must be one of: [old, prod]"
required: true
default: 'old'

jobs:
build:
Expand All @@ -31,8 +35,12 @@ jobs:
- name: Create verbosity level flags
run: |
verbosity_tags="$(echo "-$(printf "%3s" | tr " " "v")")"
echo "verbosity_tags=\"$verbosity_tags\"" >> $GITHUB_ENV
echo "verbosity_tags=$verbosity_tags" >> $GITHUB_ENV
echo "Generated verbosity flag: [$verbosity_tags]"
- name: Mapping target to inventory
run: |
echo "target_inventory=inventory/${{ github.event.inputs.target }}.ini" >> $GITHUB_ENV
echo "Added specifier for target inventory"
- name: Install ansible
run: |
sudo apt update
Expand All @@ -46,13 +54,15 @@ jobs:
sudo chmod 0600 $PRIVATE_KEY_FILE
working-directory: ${{ env.ANSIBLE_DIR }}
env:
SSH_PRIVATE_KEY: ${{ secrets.DEPLOY_KEY }}
SSH_PRIVATE_KEY: if [[ "${{ github.event.inputs.target }}" == "old" ]]; then echo ${{ secrets.OLD_DEPLOY_KEY }}; else if [[ "${{ github.event.inputs.target }}" == "prod" ]]; then echo ${{ secrets.PROD_DEPLOY_KEY }}; else echo "INVALID"; fi;
- name: Verify targets are accessible
run: ansible all --private-key=$PRIVATE_KEY_FILE -m ping
run: ansible all ${{ env.verbosity_tags }} --ssh-extra-args=${{ env.verbosity_tags }} -i ${{ env.target_inventory }} --private-key=$PRIVATE_KEY_FILE -m ping
working-directory: ${{ env.ANSIBLE_DIR }}
- name: Run playbook
run: ansible-playbook ${{ env.verbosity_tags }} --tags "$(echo "${{ github.event.inputs.tags }}" | tr -ds "[:space:]" "")" --private-key=$PRIVATE_KEY_FILE --extra-vars "ansible_sudo_pass=${{ secrets.SUDO_PASS }}" $PLAYBOOK
run: ansible-playbook ${{ env.verbosity_tags }} --ssh-extra-args=${{ env.verbosity_tags }} -i ${{ env.target_inventory }} --tags "$(echo "${{ github.event.inputs.tags }}" | tr -ds "[:space:]" "")" --private-key=$PRIVATE_KEY_FILE --extra-vars "ansible_sudo_pass=${{ env.SSH_USER_SUDO_PASS }}" $PLAYBOOK
working-directory: ${{ env.ANSIBLE_DIR }}
env:
SSH_USER_SUDO_PASS: if [[ "${{ github.event.inputs.target }}" == "old" ]]; then echo ${{ secrets.OLD_SUDO_PASS }}; else if [[ "${{ github.event.inputs.target }}" == "prod" ]]; then echo ${{ secrets.PROD_SUDO_PASS }}; else echo "INVALID"; fi;
- name: Remove key file
run: sudo rm -rf $PRIVATE_KEY_FILE
working-directory: ${{ env.ANSIBLE_DIR }}
11 changes: 10 additions & 1 deletion Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,19 @@ ENV PYTHONBUFFERED 1
RUN mkdir -p /app
WORKDIR /app

COPY . .
COPY app.py app.py
COPY fileLogHandler.py fileLogHandler.py
COPY flask_logging.conf flask_logging.conf
COPY requirements.txt requirements.txt
COPY minitask minitask
COPY knn_indexing knn_indexing
COPY summary summary
COPY summary_1 summary_1
COPY modules/RateLimiter modules/RateLimiter

RUN python3 -m pip install --upgrade pip
RUN python3 -m pip install --no-cache-dir -r requirements.txt
RUN python3 -m spacy download en_core_web_trf

EXPOSE 3001
CMD ["gunicorn", "-w", "4", "-b", "0.0.0.0:3001", "app:app"]
1 change: 0 additions & 1 deletion ansible/ansible.cfg
Original file line number Diff line number Diff line change
@@ -1,3 +1,2 @@
[defaults]
host_key_checking=False
inventory=inventories/inventory.ini
File renamed without changes.
2 changes: 2 additions & 0 deletions ansible/inventory/prod.ini
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
[targets]
ansible@65.21.139.236
2 changes: 1 addition & 1 deletion ansible/roles/backend/tasks/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@

- name: Spin up new containers
docker_compose:
project_src: "temp"
project_src: "temp/compose/backend"
build: yes
debug: yes
become: true
Expand Down
2 changes: 1 addition & 1 deletion ansible/roles/frontend/tasks/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@

- name: Spin up new containers
docker_compose:
project_src: "temp/frontend"
project_src: "temp/compose/frontend"
build: yes
debug: yes
become: true
Expand Down
Original file line number Diff line number Diff line change
@@ -1,46 +1,51 @@
- name: Stop and remove container
vars:
container_names:
- elasticsearch
- opensearch-master
- opensearch-data-ingest1
- opensearch-data-ingest2
- opensearch-data-ingest3
- opensearch-coordinator
- opensearch-dashboards
containers: "{{ item }}"
include_tasks:
file: roles/common/tasks/docker_stop_remove_tasks.yml
apply:
tags:
- elasticsearch
- opensearch
loop: "{{ container_names | batch(1) | list }}"
tags:
- elasticsearch
- opensearch

- name: Spin up new container
docker_compose:
project_src: "temp/compose/elasticsearch"
project_src: "temp/compose/opensearch"
build: true
debug: true
become: true
register: started_containers
tags:
- elasticsearch
- opensearch

- name: Wait for elasticsearch to start
command: docker exec elasticsearch curl localhost:9200/_recovery -sS
- name: Wait for opensearch to start
command: docker exec opensearch curl localhost:9200/_recovery -sS
register: result
until: result.stdout == '{}'
retries: 30
delay: 5 # Seconds
become: true
tags:
- elasticsearch
- elasticsearch_base_indexes
- elasticsearch_knn_indexes
- opensearch
- opensearch_base_indexes
- opensearch_knn_indexes

- name: Create base indexes
shell:
cmd: "python3.8 import_data.py"
chdir: "temp/minitask"
register: create_base
tags:
- elasticsearch_base_indexes
- opensearch_base_indexes

- debug: var=create_base

Expand All @@ -50,6 +55,6 @@
chdir: "temp/knn_indexing"
register: create_knn
tags:
- elasticsearch_knn_indexes
- opensearch_knn_indexes

- debug: var=create_knn
2 changes: 1 addition & 1 deletion ansible/site.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@

- hosts: targets
roles:
- elasticsearch
- opensearch

- hosts: targets
roles:
Expand Down
7 changes: 7 additions & 0 deletions caddy/Caddyfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
anu.jkl.io {
reverse_proxy localhost:3002
route /api/* {
uri strip_prefix /api
reverse_proxy localhost:3001
}
}
6 changes: 6 additions & 0 deletions caddy/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
FROM caddy:2-alpine

COPY Caddyfile /etc/caddy/Caddyfile

EXPOSE 80
EXPOSE 443
7 changes: 7 additions & 0 deletions compose/backend/docker-compose.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
version: "3.5"
services:
backend:
container_name: backend
network_mode: host
build:
context: ../../
8 changes: 8 additions & 0 deletions compose/caddy/docker-compose.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
version: "3.5"
services:
caddy:
container_name: caddy
network_mode: host
build:
context: ../../caddy
restart: unless-stopped
6 changes: 6 additions & 0 deletions compose/frontend/docker-compose.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
version: "3.5"
services:
new-frontend:
build:
context: ../../frontend_new
network_mode: host
25 changes: 6 additions & 19 deletions compose/opensearch/docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,7 @@ services:
hard: 65536
volumes:
- opensearch-master:/usr/share/opensearch/data
ports:
- 9200:9200
- 9600:9600 # required for Performance Analyzer
network_mode: host
networks:
- opensearch-net
opensearch-data-ingest1:
Expand All @@ -49,9 +47,7 @@ services:
nofile:
soft: 65536
hard: 65536
ports:
- 9201:9201
- 9601:9601 # required for Performance Analyzer
network_mode: host
volumes:
- opensearch-data-ingest1:/usr/share/opensearch/data
networks:
Expand All @@ -77,9 +73,7 @@ services:
nofile:
soft: 65536
hard: 65536
ports:
- 9202:9202
- 9602:9602 # required for Performance Analyzer
network_mode: host
volumes:
- opensearch-data-ingest2:/usr/share/opensearch/data
networks:
Expand All @@ -105,9 +99,7 @@ services:
nofile:
soft: 65536
hard: 65536
ports:
- 9203:9203
- 9603:9603 # required for Performance Analyzer
network_mode: host
volumes:
- opensearch-data-ingest2:/usr/share/opensearch/data
networks:
Expand All @@ -132,20 +124,15 @@ services:
nofile:
soft: 65536
hard: 65536
ports:
- 9204:9204
- 9604:9604 # required for Performance Analyzer
network_mode: host
volumes:
- opensearch-coordinator:/usr/share/opensearch/data
networks:
- opensearch-net
opensearch-dashboards:
image: opensearchproject/opensearch-dashboards:latest
container_name: opensearch-dashboards
ports:
- 5601:5601
expose:
- "5601"
network_mode: host
environment:
OPENSEARCH_HOSTS: '["http://opensearch-master:9200","http://opensearch-master:9201","http://opensearch-master:9202","http://opensearch-master:9203","http://opensearch-master:9204"]'
networks:
Expand Down
6 changes: 0 additions & 6 deletions compose/opensearch/opensearch.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,6 @@ network.host: 0.0.0.0
# # set to 1 to allow single node clusters
# discovery.zen.minimum_master_nodes: 1

node.name: "node-1"
# explicitly specify the node will be function as master, data and ingest
node.master: true
node.data: true
node.ingest: true

# Setting network.host to a non-loopback address enables the annoying bootstrap checks. "Single-node" mode disables them again.
#discovery.type: single-node

Expand Down
1 change: 0 additions & 1 deletion compose/redis/docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,4 +7,3 @@ services:
context: ../../modules/CuckooFilter
volumes:
- ../redis:/redis
command: redis-server /usr/local/etc/redis/redis.conf
12 changes: 12 additions & 0 deletions frontend_new/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
FROM node:lts-alpine

RUN npm install -g http-server

WORKDIR /app

COPY . .
RUN npm install
RUN npm run build

EXPOSE 3002
CMD ["http-server", "-p", "3002"]