From e92ede2962fe7023c9651cae7bf49839e8e7606d Mon Sep 17 00:00:00 2001 From: "u6940136@anu.edu.au" Date: Sat, 25 Sep 2021 15:27:30 +1000 Subject: [PATCH 01/17] :sparkles::hammer: Added new inventory and split apart to be used with -i ansible-playbook parameter --- .github/listContains.sh | 2 +- .github/workflows/ansible.yml | 12 ++++++++++-- ansible/ansible.cfg | 1 - ansible/inventories/inventory.ini | 2 -- ansible/inventory/old.ini | 2 ++ ansible/inventory/prod.ini | 2 ++ 6 files changed, 15 insertions(+), 6 deletions(-) delete mode 100644 ansible/inventories/inventory.ini create mode 100644 ansible/inventory/old.ini create mode 100644 ansible/inventory/prod.ini diff --git a/.github/listContains.sh b/.github/listContains.sh index 7238536c..567bd6c7 100644 --- a/.github/listContains.sh +++ b/.github/listContains.sh @@ -6,5 +6,5 @@ printExit() { } for value in $2; do - [[ $1 =~ (^|[[:space:]])$value($|[[:space:]]) ]] || printExit "Invlid input: $value" + [[ $1 =~ (^|[[:space:]])$value($|[[:space:]]) ]] || printExit "Invalid input: $value" done diff --git a/.github/workflows/ansible.yml b/.github/workflows/ansible.yml index e02dc0ef..41d9776b 100644 --- a/.github/workflows/ansible.yml +++ b/.github/workflows/ansible.yml @@ -11,6 +11,10 @@ on: description: "Level of verbosity to run Absible at. Must be one of: [0, 1, 2, 3]" required: true default: '1' + target: + description: "Box to target deployment to. Must be one of: [old, prod]" + required: true + default: 'old' jobs: build: @@ -33,6 +37,10 @@ jobs: verbosity_tags="$(echo "-$(printf "%3s" | tr " " "v")")" echo "verbosity_tags=\"$verbosity_tags\"" >> $GITHUB_ENV echo "Generated verbosity flag: [$verbosity_tags]" + - name: Mapping target to inventory + run: | + echo "target_inventory=\"${{ github.event.inputs.target }}\"" >> $GITHUB_ENV + echo "Added specifier for target inventory" - name: Install ansible run: | sudo apt update @@ -46,12 +54,12 @@ jobs: sudo chmod 0600 $PRIVATE_KEY_FILE working-directory: ${{ env.ANSIBLE_DIR }} env: - SSH_PRIVATE_KEY: ${{ secrets.DEPLOY_KEY }} + SSH_PRIVATE_KEY: ${{ secrets.DEPLOY_KEY }} # TODO: Update this to have both the old and prod keys and switch based on the target input - name: Verify targets are accessible run: ansible all --private-key=$PRIVATE_KEY_FILE -m ping working-directory: ${{ env.ANSIBLE_DIR }} - name: Run playbook - run: ansible-playbook ${{ env.verbosity_tags }} --tags "$(echo "${{ github.event.inputs.tags }}" | tr -ds "[:space:]" "")" --private-key=$PRIVATE_KEY_FILE --extra-vars "ansible_sudo_pass=${{ secrets.SUDO_PASS }}" $PLAYBOOK + run: ansible-playbook ${{ env.verbosity_tags }} -i ${{ env.target_inventory }} --tags "$(echo "${{ github.event.inputs.tags }}" | tr -ds "[:space:]" "")" --private-key=$PRIVATE_KEY_FILE --extra-vars "ansible_sudo_pass=${{ secrets.SUDO_PASS }}" $PLAYBOOK working-directory: ${{ env.ANSIBLE_DIR }} - name: Remove key file run: sudo rm -rf $PRIVATE_KEY_FILE diff --git a/ansible/ansible.cfg b/ansible/ansible.cfg index 6b4962d3..f288284f 100644 --- a/ansible/ansible.cfg +++ b/ansible/ansible.cfg @@ -1,3 +1,2 @@ [defaults] host_key_checking=False -inventory=inventories/inventory.ini diff --git a/ansible/inventories/inventory.ini b/ansible/inventories/inventory.ini deleted file mode 100644 index 9d867423..00000000 --- a/ansible/inventories/inventory.ini +++ /dev/null @@ -1,2 +0,0 @@ -[targets] -mike@anu.jkl.io diff --git a/ansible/inventory/old.ini b/ansible/inventory/old.ini new file mode 100644 index 00000000..f64bc6e7 --- /dev/null +++ b/ansible/inventory/old.ini @@ -0,0 +1,2 @@ +[targets] +mike@anu.jkl.io \ No newline at end of file diff --git a/ansible/inventory/prod.ini b/ansible/inventory/prod.ini new file mode 100644 index 00000000..06fc1ec6 --- /dev/null +++ b/ansible/inventory/prod.ini @@ -0,0 +1,2 @@ +[targets] +ansible@65.21.139.236 \ No newline at end of file From c22f705e28ab61e23880aa5b82f5bef8ea4d9b21 Mon Sep 17 00:00:00 2001 From: "u6940136@anu.edu.au" Date: Wed, 29 Sep 2021 16:17:51 +1000 Subject: [PATCH 02/17] :hammer: Improved frontend image serving via http-server --- caddy/Caddyfile | 7 +++++++ compose/caddy/docker-compose.yml | 14 ++++++++++++++ compose/frontend/Dockerfile | 12 ++++++++++++ compose/frontend/docker-compose.yml | 6 ++++++ 4 files changed, 39 insertions(+) create mode 100644 caddy/Caddyfile create mode 100644 compose/caddy/docker-compose.yml create mode 100644 compose/frontend/Dockerfile create mode 100644 compose/frontend/docker-compose.yml diff --git a/caddy/Caddyfile b/caddy/Caddyfile new file mode 100644 index 00000000..057947db --- /dev/null +++ b/caddy/Caddyfile @@ -0,0 +1,7 @@ +anu.jkl.io { + reverse_proxy localhost:3002 + route /api/* { + uri strip_prefix /api + reverse_proxy localhost:3001 + } +} diff --git a/compose/caddy/docker-compose.yml b/compose/caddy/docker-compose.yml new file mode 100644 index 00000000..15886b61 --- /dev/null +++ b/compose/caddy/docker-compose.yml @@ -0,0 +1,14 @@ +version: "3.5" +services: + caddy: + image: caddy/caddy:2-alpine + container_name: caddy + restart: unless-stopped + ports: + - "80:80" + - "443:443" + volumes: + - ../../caddy/Caddyfile:/etc/caddy/Caddyfile + # These start out empty and fill with cache data + # - caddy/data:/data + # - caddy/config:/config diff --git a/compose/frontend/Dockerfile b/compose/frontend/Dockerfile new file mode 100644 index 00000000..3a55d3ef --- /dev/null +++ b/compose/frontend/Dockerfile @@ -0,0 +1,12 @@ +FROM node:lts-alpine + +RUN npm install -g http-server + +WORKDIR /app + +COPY ../../frontend_new . +RUN npm install +RUN npm run build + +EXPOSE 3002 +CMD ["http-server", "-p", "3002"] \ No newline at end of file diff --git a/compose/frontend/docker-compose.yml b/compose/frontend/docker-compose.yml new file mode 100644 index 00000000..7d2bca33 --- /dev/null +++ b/compose/frontend/docker-compose.yml @@ -0,0 +1,6 @@ +version: "3.5" +services: + new-frontend: + build: . + ports: + - "3002:3002" From b86e7b2848ed0863430ff1ba190fcdcd8c298988 Mon Sep 17 00:00:00 2001 From: "u6940136@anu.edu.au" Date: Thu, 30 Sep 2021 12:22:33 +1000 Subject: [PATCH 03/17] :hammer: Switched env vars based on whether old or prod has been selected --- .github/workflows/ansible.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ansible.yml b/.github/workflows/ansible.yml index 41d9776b..e5a3f34f 100644 --- a/.github/workflows/ansible.yml +++ b/.github/workflows/ansible.yml @@ -54,13 +54,15 @@ jobs: sudo chmod 0600 $PRIVATE_KEY_FILE working-directory: ${{ env.ANSIBLE_DIR }} env: - SSH_PRIVATE_KEY: ${{ secrets.DEPLOY_KEY }} # TODO: Update this to have both the old and prod keys and switch based on the target input + SSH_PRIVATE_KEY: if [[ ${{ github.event.inputs.target }} == "old" ]]; then ${{ secrets.OLD_DEPLOY_KEY }}; else if [[ ${{ github.event.inputs.target }} == "prod" ]]; then ${{ secrets.PROD_DEPLOY_KEY }}; else "INVALID"; fi; - name: Verify targets are accessible run: ansible all --private-key=$PRIVATE_KEY_FILE -m ping working-directory: ${{ env.ANSIBLE_DIR }} - name: Run playbook - run: ansible-playbook ${{ env.verbosity_tags }} -i ${{ env.target_inventory }} --tags "$(echo "${{ github.event.inputs.tags }}" | tr -ds "[:space:]" "")" --private-key=$PRIVATE_KEY_FILE --extra-vars "ansible_sudo_pass=${{ secrets.SUDO_PASS }}" $PLAYBOOK + run: ansible-playbook ${{ env.verbosity_tags }} -i ${{ env.target_inventory }} --tags "$(echo "${{ github.event.inputs.tags }}" | tr -ds "[:space:]" "")" --private-key=$PRIVATE_KEY_FILE --extra-vars "ansible_sudo_pass=$SUDO_PASS" $PLAYBOOK working-directory: ${{ env.ANSIBLE_DIR }} + env: + SUDO_PASS: if [[ ${{ github.event.inputs.target }} == "old" ]]; then ${{ secrets.OLD_SUDO_PASS }}; else if [[ ${{ github.event.inputs.target }} == "prod" ]]; then ${{ secrets.PROD_SUDO_PASS }}; else "INVALID"; fi; - name: Remove key file run: sudo rm -rf $PRIVATE_KEY_FILE working-directory: ${{ env.ANSIBLE_DIR }} From 8e2e18bd0f285985271f37ed696465d03ae73f58 Mon Sep 17 00:00:00 2001 From: "u6940136@anu.edu.au" Date: Thu, 30 Sep 2021 15:07:47 +1000 Subject: [PATCH 04/17] :hammer: Moved private key location and removed redundant password usage --- .github/workflows/ansible.yml | 12 +++++------- compose/caddy/docker-compose.yml | 3 --- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ansible.yml b/.github/workflows/ansible.yml index e5a3f34f..292e828e 100644 --- a/.github/workflows/ansible.yml +++ b/.github/workflows/ansible.yml @@ -20,7 +20,7 @@ jobs: build: runs-on: ubuntu-latest env: - PRIVATE_KEY_FILE: id_rsa + PRIVATE_KEY_FILE: ~/.ssh/prod_is_rsa ANSIBLE_DIR: ./ansible PLAYBOOK: site.yml steps: @@ -39,7 +39,7 @@ jobs: echo "Generated verbosity flag: [$verbosity_tags]" - name: Mapping target to inventory run: | - echo "target_inventory=\"${{ github.event.inputs.target }}\"" >> $GITHUB_ENV + echo "target_inventory=\"inventory/${{ github.event.inputs.target }}.ini\"" >> $GITHUB_ENV echo "Added specifier for target inventory" - name: Install ansible run: | @@ -54,15 +54,13 @@ jobs: sudo chmod 0600 $PRIVATE_KEY_FILE working-directory: ${{ env.ANSIBLE_DIR }} env: - SSH_PRIVATE_KEY: if [[ ${{ github.event.inputs.target }} == "old" ]]; then ${{ secrets.OLD_DEPLOY_KEY }}; else if [[ ${{ github.event.inputs.target }} == "prod" ]]; then ${{ secrets.PROD_DEPLOY_KEY }}; else "INVALID"; fi; + SSH_PRIVATE_KEY: if [[ "${{ github.event.inputs.target }}" == "old" ]]; then echo ${{ secrets.OLD_DEPLOY_KEY }}; else if [[ "${{ github.event.inputs.target }}" == "prod" ]]; then echo ${{ secrets.PROD_DEPLOY_KEY }}; else echo "INVALID"; fi; - name: Verify targets are accessible - run: ansible all --private-key=$PRIVATE_KEY_FILE -m ping + run: ansible all ${{ env.verbosity_tags }} -i ${{ env.target_inventory }} --private-key=$PRIVATE_KEY_FILE -m ping working-directory: ${{ env.ANSIBLE_DIR }} - name: Run playbook - run: ansible-playbook ${{ env.verbosity_tags }} -i ${{ env.target_inventory }} --tags "$(echo "${{ github.event.inputs.tags }}" | tr -ds "[:space:]" "")" --private-key=$PRIVATE_KEY_FILE --extra-vars "ansible_sudo_pass=$SUDO_PASS" $PLAYBOOK + run: ansible-playbook ${{ env.verbosity_tags }} -i ${{ env.target_inventory }} --tags "$(echo "${{ github.event.inputs.tags }}" | tr -ds "[:space:]" "")" --private-key=$PRIVATE_KEY_FILE $PLAYBOOK working-directory: ${{ env.ANSIBLE_DIR }} - env: - SUDO_PASS: if [[ ${{ github.event.inputs.target }} == "old" ]]; then ${{ secrets.OLD_SUDO_PASS }}; else if [[ ${{ github.event.inputs.target }} == "prod" ]]; then ${{ secrets.PROD_SUDO_PASS }}; else "INVALID"; fi; - name: Remove key file run: sudo rm -rf $PRIVATE_KEY_FILE working-directory: ${{ env.ANSIBLE_DIR }} diff --git a/compose/caddy/docker-compose.yml b/compose/caddy/docker-compose.yml index 15886b61..61543617 100644 --- a/compose/caddy/docker-compose.yml +++ b/compose/caddy/docker-compose.yml @@ -9,6 +9,3 @@ services: - "443:443" volumes: - ../../caddy/Caddyfile:/etc/caddy/Caddyfile - # These start out empty and fill with cache data - # - caddy/data:/data - # - caddy/config:/config From 1ec0eab8d6a8be1b09d1e27874751d247f85805c Mon Sep 17 00:00:00 2001 From: "u6940136@anu.edu.au" Date: Thu, 30 Sep 2021 15:10:43 +1000 Subject: [PATCH 05/17] :hammer: Moved private key location and removed redundant password usage --- .github/workflows/ansible.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ansible.yml b/.github/workflows/ansible.yml index 292e828e..e0dfbccc 100644 --- a/.github/workflows/ansible.yml +++ b/.github/workflows/ansible.yml @@ -20,7 +20,7 @@ jobs: build: runs-on: ubuntu-latest env: - PRIVATE_KEY_FILE: ~/.ssh/prod_is_rsa + PRIVATE_KEY_FILE: id_rsa ANSIBLE_DIR: ./ansible PLAYBOOK: site.yml steps: From 1050240bb367ad9abccd9f93a1fc355ff457b99a Mon Sep 17 00:00:00 2001 From: "u6940136@anu.edu.au" Date: Sun, 3 Oct 2021 17:15:32 +1100 Subject: [PATCH 06/17] :hammer: Re-added sudo pass via args --- .github/workflows/ansible.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ansible.yml b/.github/workflows/ansible.yml index e0dfbccc..46c2b845 100644 --- a/.github/workflows/ansible.yml +++ b/.github/workflows/ansible.yml @@ -59,8 +59,10 @@ jobs: run: ansible all ${{ env.verbosity_tags }} -i ${{ env.target_inventory }} --private-key=$PRIVATE_KEY_FILE -m ping working-directory: ${{ env.ANSIBLE_DIR }} - name: Run playbook - run: ansible-playbook ${{ env.verbosity_tags }} -i ${{ env.target_inventory }} --tags "$(echo "${{ github.event.inputs.tags }}" | tr -ds "[:space:]" "")" --private-key=$PRIVATE_KEY_FILE $PLAYBOOK + run: ansible-playbook ${{ env.verbosity_tags }} -i ${{ env.target_inventory }} --tags "$(echo "${{ github.event.inputs.tags }}" | tr -ds "[:space:]" "")" --private-key=$PRIVATE_KEY_FILE --extra-vars "ansible_sudo_pass=${{ env.SUDO_PASS }}" $PLAYBOOK working-directory: ${{ env.ANSIBLE_DIR }} + env: + SUDO_PASS: if [[ "${{ github.event.inputs.target }}" == "old" ]]; then echo ${{ secrets.OLD_SUDO_PASS }}; else if [[ "${{ github.event.inputs.target }}" == "prod" ]]; then echo ${{ secrets.PROD_SUDO_PASS }}; else echo "INVALID"; fi; - name: Remove key file run: sudo rm -rf $PRIVATE_KEY_FILE working-directory: ${{ env.ANSIBLE_DIR }} From 71feff706cf170ca884deceab0f17f600e478fc4 Mon Sep 17 00:00:00 2001 From: "u6940136@anu.edu.au" Date: Sun, 3 Oct 2021 17:54:56 +1100 Subject: [PATCH 07/17] :hammer: Updated workflow and targets --- .github/workflows/ansible.yml | 6 ++---- ansible/inventory/old.ini | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ansible.yml b/.github/workflows/ansible.yml index 46c2b845..e45065f4 100644 --- a/.github/workflows/ansible.yml +++ b/.github/workflows/ansible.yml @@ -56,13 +56,11 @@ jobs: env: SSH_PRIVATE_KEY: if [[ "${{ github.event.inputs.target }}" == "old" ]]; then echo ${{ secrets.OLD_DEPLOY_KEY }}; else if [[ "${{ github.event.inputs.target }}" == "prod" ]]; then echo ${{ secrets.PROD_DEPLOY_KEY }}; else echo "INVALID"; fi; - name: Verify targets are accessible - run: ansible all ${{ env.verbosity_tags }} -i ${{ env.target_inventory }} --private-key=$PRIVATE_KEY_FILE -m ping + run: ansible all ${{ env.verbosity_tags }} --ssh-extra-args=${{ env.verbosity_tags }} -i ${{ env.target_inventory }} --private-key=$PRIVATE_KEY_FILE -m ping working-directory: ${{ env.ANSIBLE_DIR }} - name: Run playbook - run: ansible-playbook ${{ env.verbosity_tags }} -i ${{ env.target_inventory }} --tags "$(echo "${{ github.event.inputs.tags }}" | tr -ds "[:space:]" "")" --private-key=$PRIVATE_KEY_FILE --extra-vars "ansible_sudo_pass=${{ env.SUDO_PASS }}" $PLAYBOOK + run: ansible-playbook ${{ env.verbosity_tags }} --ssh-extra-args=${{ env.verbosity_tags }} -i ${{ env.target_inventory }} --tags "$(echo "${{ github.event.inputs.tags }}" | tr -ds "[:space:]" "")" --private-key=$PRIVATE_KEY_FILE $PLAYBOOK working-directory: ${{ env.ANSIBLE_DIR }} - env: - SUDO_PASS: if [[ "${{ github.event.inputs.target }}" == "old" ]]; then echo ${{ secrets.OLD_SUDO_PASS }}; else if [[ "${{ github.event.inputs.target }}" == "prod" ]]; then echo ${{ secrets.PROD_SUDO_PASS }}; else echo "INVALID"; fi; - name: Remove key file run: sudo rm -rf $PRIVATE_KEY_FILE working-directory: ${{ env.ANSIBLE_DIR }} diff --git a/ansible/inventory/old.ini b/ansible/inventory/old.ini index f64bc6e7..18ab141c 100644 --- a/ansible/inventory/old.ini +++ b/ansible/inventory/old.ini @@ -1,2 +1,2 @@ [targets] -mike@anu.jkl.io \ No newline at end of file +ansible@anu.jkl.io \ No newline at end of file From 79c681d8891d13aa0b06f4d752f251fa2ab9d1cf Mon Sep 17 00:00:00 2001 From: "u6940136@anu.edu.au" Date: Sun, 3 Oct 2021 17:57:49 +1100 Subject: [PATCH 08/17] :hammer: Added user sudo pass to be switched by target --- .github/workflows/ansible.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ansible.yml b/.github/workflows/ansible.yml index e45065f4..40373b74 100644 --- a/.github/workflows/ansible.yml +++ b/.github/workflows/ansible.yml @@ -59,8 +59,10 @@ jobs: run: ansible all ${{ env.verbosity_tags }} --ssh-extra-args=${{ env.verbosity_tags }} -i ${{ env.target_inventory }} --private-key=$PRIVATE_KEY_FILE -m ping working-directory: ${{ env.ANSIBLE_DIR }} - name: Run playbook - run: ansible-playbook ${{ env.verbosity_tags }} --ssh-extra-args=${{ env.verbosity_tags }} -i ${{ env.target_inventory }} --tags "$(echo "${{ github.event.inputs.tags }}" | tr -ds "[:space:]" "")" --private-key=$PRIVATE_KEY_FILE $PLAYBOOK + run: ansible-playbook ${{ env.verbosity_tags }} --ssh-extra-args=${{ env.verbosity_tags }} -i ${{ env.target_inventory }} --tags "$(echo "${{ github.event.inputs.tags }}" | tr -ds "[:space:]" "")" --private-key=$PRIVATE_KEY_FILE --extra-vars "ansible_sudo_pass=${{ env.SSH_USER_SUDO_PASS }}" $PLAYBOOK working-directory: ${{ env.ANSIBLE_DIR }} + env: + SSH_USER_SUDO_PASS: if [[ "${{ github.event.inputs.target }}" == "old" ]]; then echo ${{ secrets.OLD_SUDO_PASS }}; else if [[ "${{ github.event.inputs.target }}" == "prod" ]]; then echo ${{ secrets.PROD_SUDO_PASS }}; else echo "INVALID"; fi; - name: Remove key file run: sudo rm -rf $PRIVATE_KEY_FILE working-directory: ${{ env.ANSIBLE_DIR }} From 9949759e0e3a7c1ce17081bd90539997e548a8f0 Mon Sep 17 00:00:00 2001 From: "u6940136@anu.edu.au" Date: Sun, 3 Oct 2021 18:00:17 +1100 Subject: [PATCH 09/17] :sparkled: Added caddy deployment steps in Ansible playbook --- .github/workflows/ansible.yml | 2 +- ansible/roles/caddy/main.yml | 23 +++++++++++++++++++++++ ansible/site.yml | 4 ++++ 3 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 ansible/roles/caddy/main.yml diff --git a/.github/workflows/ansible.yml b/.github/workflows/ansible.yml index 40373b74..d74b2ceb 100644 --- a/.github/workflows/ansible.yml +++ b/.github/workflows/ansible.yml @@ -4,7 +4,7 @@ on: workflow_dispatch: inputs: tags: - description: 'Comma seperated list of components to deploy from: [elasticsearch, elasticsearch_base_indexes, elasticsearch_knn_indexes, redis, redis_backup_restore, frontend, backend]' + description: 'Comma seperated list of components to deploy from: [caddy, elasticsearch, elasticsearch_base_indexes, elasticsearch_knn_indexes, redis, redis_backup_restore, frontend, backend]' required: true default: 'elasticsearch, elasticsearch_base_indexes, elasticsearch_knn_indexes, redis, redis_backup_restore, frontend, backend' verbosity: diff --git a/ansible/roles/caddy/main.yml b/ansible/roles/caddy/main.yml new file mode 100644 index 00000000..9b92cf62 --- /dev/null +++ b/ansible/roles/caddy/main.yml @@ -0,0 +1,23 @@ +- name: Docker stop and remove containers + vars: + container_names: + - caddy + containers: "{{ item }}" + include_tasks: + file: roles/common/tasks/docker_stop_remove_tasks.yml + apply: + tags: + - caddy + loop: "{{ container_names | batch(1) | list }}" + tags: + - caddy + +- name: Spin up new containers + docker_compose: + project_src: "temp" + build: yes + debug: yes + become: true + register: started_containers + tags: + - caddy diff --git a/ansible/site.yml b/ansible/site.yml index a566781c..54030c66 100644 --- a/ansible/site.yml +++ b/ansible/site.yml @@ -15,6 +15,10 @@ tags: - always +- hosts: targets + roles: + - caddy + - hosts: targets roles: - elasticsearch From 5ed2524f2a00057e165add7c2ba00af082de7c52 Mon Sep 17 00:00:00 2001 From: "u6940136@anu.edu.au" Date: Sun, 3 Oct 2021 18:02:30 +1100 Subject: [PATCH 10/17] :hammer: Refactored frontend deployment to target new location and Dockerfile --- ansible/roles/caddy/main.yml | 2 +- ansible/roles/frontend/tasks/main.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/roles/caddy/main.yml b/ansible/roles/caddy/main.yml index 9b92cf62..36c0fd65 100644 --- a/ansible/roles/caddy/main.yml +++ b/ansible/roles/caddy/main.yml @@ -14,7 +14,7 @@ - name: Spin up new containers docker_compose: - project_src: "temp" + project_src: "temp/compose/caddy" build: yes debug: yes become: true diff --git a/ansible/roles/frontend/tasks/main.yml b/ansible/roles/frontend/tasks/main.yml index e3e8d1fc..effcd27e 100644 --- a/ansible/roles/frontend/tasks/main.yml +++ b/ansible/roles/frontend/tasks/main.yml @@ -14,7 +14,7 @@ - name: Spin up new containers docker_compose: - project_src: "temp/frontend" + project_src: "temp/compose/frontend" build: yes debug: yes become: true From ba0e4b2101cff48a9799c9b64b6e639476210a72 Mon Sep 17 00:00:00 2001 From: "u6940136@anu.edu.au" Date: Tue, 5 Oct 2021 21:17:17 +1100 Subject: [PATCH 11/17] :hammer::bug::sparkles: Fixed dockerfiles, added more strict copy for backend and addressed directory differences with backend role --- .github/workflows/ansible.yml | 6 ++--- Dockerfile | 11 ++++++++- ansible/roles/backend/tasks/main.yml | 2 +- ansible/roles/caddy/main.yml | 23 ------------------- ansible/site.yml | 4 ---- caddy/Dockerfile | 6 +++++ compose/backend/docker-compose.yml | 7 ++++++ compose/caddy/docker-compose.yml | 9 +++----- compose/elasticsearch/Dockerfile | 5 +++- compose/frontend/docker-compose.yml | 6 ++--- compose/redis/docker-compose.yml | 1 - {compose/frontend => frontend_new}/Dockerfile | 2 +- 12 files changed, 38 insertions(+), 44 deletions(-) delete mode 100644 ansible/roles/caddy/main.yml create mode 100644 caddy/Dockerfile create mode 100644 compose/backend/docker-compose.yml rename {compose/frontend => frontend_new}/Dockerfile (66%) diff --git a/.github/workflows/ansible.yml b/.github/workflows/ansible.yml index d74b2ceb..120dc7bb 100644 --- a/.github/workflows/ansible.yml +++ b/.github/workflows/ansible.yml @@ -4,7 +4,7 @@ on: workflow_dispatch: inputs: tags: - description: 'Comma seperated list of components to deploy from: [caddy, elasticsearch, elasticsearch_base_indexes, elasticsearch_knn_indexes, redis, redis_backup_restore, frontend, backend]' + description: 'Comma seperated list of components to deploy from: [elasticsearch, elasticsearch_base_indexes, elasticsearch_knn_indexes, redis, redis_backup_restore, frontend, backend]' required: true default: 'elasticsearch, elasticsearch_base_indexes, elasticsearch_knn_indexes, redis, redis_backup_restore, frontend, backend' verbosity: @@ -35,11 +35,11 @@ jobs: - name: Create verbosity level flags run: | verbosity_tags="$(echo "-$(printf "%3s" | tr " " "v")")" - echo "verbosity_tags=\"$verbosity_tags\"" >> $GITHUB_ENV + echo "verbosity_tags=$verbosity_tags" >> $GITHUB_ENV echo "Generated verbosity flag: [$verbosity_tags]" - name: Mapping target to inventory run: | - echo "target_inventory=\"inventory/${{ github.event.inputs.target }}.ini\"" >> $GITHUB_ENV + echo "target_inventory=inventory/${{ github.event.inputs.target }}.ini" >> $GITHUB_ENV echo "Added specifier for target inventory" - name: Install ansible run: | diff --git a/Dockerfile b/Dockerfile index ec2b54bb..d7195cdb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,10 +4,19 @@ ENV PYTHONBUFFERED 1 RUN mkdir -p /app WORKDIR /app -COPY . . +COPY app.py app.py +COPY fileLogHandler.py fileLogHandler.py +COPY flask_logging.conf flask_logging.conf +COPY requirements.txt requirements.txt +COPY minitask minitask +COPY knn_indexing knn_indexing +COPY summary summary +COPY summary_1 summary_1 +copy modules/RateLimiter modules/RateLimiter RUN python3 -m pip install --upgrade pip RUN python3 -m pip install --no-cache-dir -r requirements.txt RUN python3 -m spacy download en_core_web_trf EXPOSE 3001 +CMD ["gunicorn", "-w", "4", "-b", "0.0.0.0:3001", "app:app"] \ No newline at end of file diff --git a/ansible/roles/backend/tasks/main.yml b/ansible/roles/backend/tasks/main.yml index 69eb5f7f..174e4467 100644 --- a/ansible/roles/backend/tasks/main.yml +++ b/ansible/roles/backend/tasks/main.yml @@ -14,7 +14,7 @@ - name: Spin up new containers docker_compose: - project_src: "temp" + project_src: "temp/compose/backend" build: yes debug: yes become: true diff --git a/ansible/roles/caddy/main.yml b/ansible/roles/caddy/main.yml deleted file mode 100644 index 36c0fd65..00000000 --- a/ansible/roles/caddy/main.yml +++ /dev/null @@ -1,23 +0,0 @@ -- name: Docker stop and remove containers - vars: - container_names: - - caddy - containers: "{{ item }}" - include_tasks: - file: roles/common/tasks/docker_stop_remove_tasks.yml - apply: - tags: - - caddy - loop: "{{ container_names | batch(1) | list }}" - tags: - - caddy - -- name: Spin up new containers - docker_compose: - project_src: "temp/compose/caddy" - build: yes - debug: yes - become: true - register: started_containers - tags: - - caddy diff --git a/ansible/site.yml b/ansible/site.yml index 54030c66..a566781c 100644 --- a/ansible/site.yml +++ b/ansible/site.yml @@ -15,10 +15,6 @@ tags: - always -- hosts: targets - roles: - - caddy - - hosts: targets roles: - elasticsearch diff --git a/caddy/Dockerfile b/caddy/Dockerfile new file mode 100644 index 00000000..9a0f4937 --- /dev/null +++ b/caddy/Dockerfile @@ -0,0 +1,6 @@ +FROM caddy:2-alpine + +COPY Caddyfile /etc/caddy/Caddyfile + +EXPOSE 80 +EXPOSE 443 \ No newline at end of file diff --git a/compose/backend/docker-compose.yml b/compose/backend/docker-compose.yml new file mode 100644 index 00000000..ac1df549 --- /dev/null +++ b/compose/backend/docker-compose.yml @@ -0,0 +1,7 @@ +version: "3.5" +services: + backend: + container_name: backend + network_mode: host + build: + context: ../../ diff --git a/compose/caddy/docker-compose.yml b/compose/caddy/docker-compose.yml index 61543617..332d2cc8 100644 --- a/compose/caddy/docker-compose.yml +++ b/compose/caddy/docker-compose.yml @@ -1,11 +1,8 @@ version: "3.5" services: caddy: - image: caddy/caddy:2-alpine container_name: caddy + network_mode: host + build: + context: ../../caddy restart: unless-stopped - ports: - - "80:80" - - "443:443" - volumes: - - ../../caddy/Caddyfile:/etc/caddy/Caddyfile diff --git a/compose/elasticsearch/Dockerfile b/compose/elasticsearch/Dockerfile index ac89fe47..86a140a1 100644 --- a/compose/elasticsearch/Dockerfile +++ b/compose/elasticsearch/Dockerfile @@ -1,3 +1,6 @@ FROM opensearchproject/opensearch:latest -COPY opensearch.yml /usr/share/opensearch/config/opensearch.yml \ No newline at end of file +COPY opensearch.yml /usr/share/opensearch/config/opensearch.yml + +EXPOSE 9200 +EXPOSE 9300 \ No newline at end of file diff --git a/compose/frontend/docker-compose.yml b/compose/frontend/docker-compose.yml index 7d2bca33..f784a1c6 100644 --- a/compose/frontend/docker-compose.yml +++ b/compose/frontend/docker-compose.yml @@ -1,6 +1,6 @@ version: "3.5" services: new-frontend: - build: . - ports: - - "3002:3002" + build: + context: ../../frontend_new + network_mode: host diff --git a/compose/redis/docker-compose.yml b/compose/redis/docker-compose.yml index edb9be84..e66de534 100644 --- a/compose/redis/docker-compose.yml +++ b/compose/redis/docker-compose.yml @@ -7,4 +7,3 @@ services: context: ../../modules/CuckooFilter volumes: - ../redis:/redis - command: redis-server /usr/local/etc/redis/redis.conf diff --git a/compose/frontend/Dockerfile b/frontend_new/Dockerfile similarity index 66% rename from compose/frontend/Dockerfile rename to frontend_new/Dockerfile index 3a55d3ef..34762195 100644 --- a/compose/frontend/Dockerfile +++ b/frontend_new/Dockerfile @@ -4,7 +4,7 @@ RUN npm install -g http-server WORKDIR /app -COPY ../../frontend_new . +COPY . . RUN npm install RUN npm run build From 6aef8f0ca90426587442f14f0d63e2e439e16013 Mon Sep 17 00:00:00 2001 From: "u6940136@anu.edu.au" Date: Tue, 5 Oct 2021 21:32:00 +1100 Subject: [PATCH 12/17] :hammer: Made copy uppercase in backend dockerfile for consistency --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index d7195cdb..c222facb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,7 @@ COPY minitask minitask COPY knn_indexing knn_indexing COPY summary summary COPY summary_1 summary_1 -copy modules/RateLimiter modules/RateLimiter +COPY modules/RateLimiter modules/RateLimiter RUN python3 -m pip install --upgrade pip RUN python3 -m pip install --no-cache-dir -r requirements.txt From 31a171fd651cfbdcca75fce05e1ad87915cde838 Mon Sep 17 00:00:00 2001 From: EngineersBox <35655145+EngineersBox@users.noreply.github.com> Date: Wed, 6 Oct 2021 16:22:26 +1100 Subject: [PATCH 13/17] Added newline --- frontend_new/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend_new/Dockerfile b/frontend_new/Dockerfile index 34762195..5d743098 100644 --- a/frontend_new/Dockerfile +++ b/frontend_new/Dockerfile @@ -9,4 +9,4 @@ RUN npm install RUN npm run build EXPOSE 3002 -CMD ["http-server", "-p", "3002"] \ No newline at end of file +CMD ["http-server", "-p", "3002"] From ba4068f6e8fd421b34e6546ea9dee136ab77c283 Mon Sep 17 00:00:00 2001 From: EngineersBox <35655145+EngineersBox@users.noreply.github.com> Date: Wed, 6 Oct 2021 16:22:52 +1100 Subject: [PATCH 14/17] Added newline --- caddy/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/caddy/Dockerfile b/caddy/Dockerfile index 9a0f4937..200a1aa6 100644 --- a/caddy/Dockerfile +++ b/caddy/Dockerfile @@ -3,4 +3,4 @@ FROM caddy:2-alpine COPY Caddyfile /etc/caddy/Caddyfile EXPOSE 80 -EXPOSE 443 \ No newline at end of file +EXPOSE 443 From a32d7b468be4f96a7c764838b9503e62aa878359 Mon Sep 17 00:00:00 2001 From: EngineersBox <35655145+EngineersBox@users.noreply.github.com> Date: Wed, 6 Oct 2021 16:23:09 +1100 Subject: [PATCH 15/17] Added newline --- ansible/inventory/prod.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/inventory/prod.ini b/ansible/inventory/prod.ini index 06fc1ec6..50592e23 100644 --- a/ansible/inventory/prod.ini +++ b/ansible/inventory/prod.ini @@ -1,2 +1,2 @@ [targets] -ansible@65.21.139.236 \ No newline at end of file +ansible@65.21.139.236 From c349a8f67bccbe2540fcd02cda883ae8c5ec5141 Mon Sep 17 00:00:00 2001 From: EngineersBox <35655145+EngineersBox@users.noreply.github.com> Date: Wed, 6 Oct 2021 16:23:20 +1100 Subject: [PATCH 16/17] Added newline --- ansible/inventory/old.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/inventory/old.ini b/ansible/inventory/old.ini index 18ab141c..051e9f2e 100644 --- a/ansible/inventory/old.ini +++ b/ansible/inventory/old.ini @@ -1,2 +1,2 @@ [targets] -ansible@anu.jkl.io \ No newline at end of file +ansible@anu.jkl.io From 1ff930ffc79e3089c20f12bb6115a1660a07efb1 Mon Sep 17 00:00:00 2001 From: EngineersBox <35655145+EngineersBox@users.noreply.github.com> Date: Wed, 6 Oct 2021 16:23:33 +1100 Subject: [PATCH 17/17] Added newline --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index c222facb..6e6ae0c0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,4 +19,4 @@ RUN python3 -m pip install --no-cache-dir -r requirements.txt RUN python3 -m spacy download en_core_web_trf EXPOSE 3001 -CMD ["gunicorn", "-w", "4", "-b", "0.0.0.0:3001", "app:app"] \ No newline at end of file +CMD ["gunicorn", "-w", "4", "-b", "0.0.0.0:3001", "app:app"]