diff --git a/.github/listContains.sh b/.github/listContains.sh index 7238536c..567bd6c7 100644 --- a/.github/listContains.sh +++ b/.github/listContains.sh @@ -6,5 +6,5 @@ printExit() { } for value in $2; do - [[ $1 =~ (^|[[:space:]])$value($|[[:space:]]) ]] || printExit "Invlid input: $value" + [[ $1 =~ (^|[[:space:]])$value($|[[:space:]]) ]] || printExit "Invalid input: $value" done diff --git a/.github/workflows/ansible.yml b/.github/workflows/ansible.yml index e02dc0ef..120dc7bb 100644 --- a/.github/workflows/ansible.yml +++ b/.github/workflows/ansible.yml @@ -11,6 +11,10 @@ on: description: "Level of verbosity to run Absible at. Must be one of: [0, 1, 2, 3]" required: true default: '1' + target: + description: "Box to target deployment to. Must be one of: [old, prod]" + required: true + default: 'old' jobs: build: @@ -31,8 +35,12 @@ jobs: - name: Create verbosity level flags run: | verbosity_tags="$(echo "-$(printf "%3s" | tr " " "v")")" - echo "verbosity_tags=\"$verbosity_tags\"" >> $GITHUB_ENV + echo "verbosity_tags=$verbosity_tags" >> $GITHUB_ENV echo "Generated verbosity flag: [$verbosity_tags]" + - name: Mapping target to inventory + run: | + echo "target_inventory=inventory/${{ github.event.inputs.target }}.ini" >> $GITHUB_ENV + echo "Added specifier for target inventory" - name: Install ansible run: | sudo apt update @@ -46,13 +54,15 @@ jobs: sudo chmod 0600 $PRIVATE_KEY_FILE working-directory: ${{ env.ANSIBLE_DIR }} env: - SSH_PRIVATE_KEY: ${{ secrets.DEPLOY_KEY }} + SSH_PRIVATE_KEY: if [[ "${{ github.event.inputs.target }}" == "old" ]]; then echo ${{ secrets.OLD_DEPLOY_KEY }}; else if [[ "${{ github.event.inputs.target }}" == "prod" ]]; then echo ${{ secrets.PROD_DEPLOY_KEY }}; else echo "INVALID"; fi; - name: Verify targets are accessible - run: ansible all --private-key=$PRIVATE_KEY_FILE -m ping + run: ansible all ${{ env.verbosity_tags }} --ssh-extra-args=${{ env.verbosity_tags }} -i ${{ env.target_inventory }} --private-key=$PRIVATE_KEY_FILE -m ping working-directory: ${{ env.ANSIBLE_DIR }} - name: Run playbook - run: ansible-playbook ${{ env.verbosity_tags }} --tags "$(echo "${{ github.event.inputs.tags }}" | tr -ds "[:space:]" "")" --private-key=$PRIVATE_KEY_FILE --extra-vars "ansible_sudo_pass=${{ secrets.SUDO_PASS }}" $PLAYBOOK + run: ansible-playbook ${{ env.verbosity_tags }} --ssh-extra-args=${{ env.verbosity_tags }} -i ${{ env.target_inventory }} --tags "$(echo "${{ github.event.inputs.tags }}" | tr -ds "[:space:]" "")" --private-key=$PRIVATE_KEY_FILE --extra-vars "ansible_sudo_pass=${{ env.SSH_USER_SUDO_PASS }}" $PLAYBOOK working-directory: ${{ env.ANSIBLE_DIR }} + env: + SSH_USER_SUDO_PASS: if [[ "${{ github.event.inputs.target }}" == "old" ]]; then echo ${{ secrets.OLD_SUDO_PASS }}; else if [[ "${{ github.event.inputs.target }}" == "prod" ]]; then echo ${{ secrets.PROD_SUDO_PASS }}; else echo "INVALID"; fi; - name: Remove key file run: sudo rm -rf $PRIVATE_KEY_FILE working-directory: ${{ env.ANSIBLE_DIR }} diff --git a/Dockerfile b/Dockerfile index ec2b54bb..6e6ae0c0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,10 +4,19 @@ ENV PYTHONBUFFERED 1 RUN mkdir -p /app WORKDIR /app -COPY . . +COPY app.py app.py +COPY fileLogHandler.py fileLogHandler.py +COPY flask_logging.conf flask_logging.conf +COPY requirements.txt requirements.txt +COPY minitask minitask +COPY knn_indexing knn_indexing +COPY summary summary +COPY summary_1 summary_1 +COPY modules/RateLimiter modules/RateLimiter RUN python3 -m pip install --upgrade pip RUN python3 -m pip install --no-cache-dir -r requirements.txt RUN python3 -m spacy download en_core_web_trf EXPOSE 3001 +CMD ["gunicorn", "-w", "4", "-b", "0.0.0.0:3001", "app:app"] diff --git a/ansible/ansible.cfg b/ansible/ansible.cfg index 6b4962d3..f288284f 100644 --- a/ansible/ansible.cfg +++ b/ansible/ansible.cfg @@ -1,3 +1,2 @@ [defaults] host_key_checking=False -inventory=inventories/inventory.ini diff --git a/ansible/inventories/inventory.ini b/ansible/inventory/old.ini similarity index 100% rename from ansible/inventories/inventory.ini rename to ansible/inventory/old.ini diff --git a/ansible/inventory/prod.ini b/ansible/inventory/prod.ini new file mode 100644 index 00000000..50592e23 --- /dev/null +++ b/ansible/inventory/prod.ini @@ -0,0 +1,2 @@ +[targets] +ansible@65.21.139.236 diff --git a/ansible/roles/backend/tasks/main.yml b/ansible/roles/backend/tasks/main.yml index 69eb5f7f..174e4467 100644 --- a/ansible/roles/backend/tasks/main.yml +++ b/ansible/roles/backend/tasks/main.yml @@ -14,7 +14,7 @@ - name: Spin up new containers docker_compose: - project_src: "temp" + project_src: "temp/compose/backend" build: yes debug: yes become: true diff --git a/ansible/roles/frontend/tasks/main.yml b/ansible/roles/frontend/tasks/main.yml index e3e8d1fc..effcd27e 100644 --- a/ansible/roles/frontend/tasks/main.yml +++ b/ansible/roles/frontend/tasks/main.yml @@ -14,7 +14,7 @@ - name: Spin up new containers docker_compose: - project_src: "temp/frontend" + project_src: "temp/compose/frontend" build: yes debug: yes become: true diff --git a/ansible/roles/elasticsearch/tasks/main.yml b/ansible/roles/opensearch/tasks/main.yml similarity index 60% rename from ansible/roles/elasticsearch/tasks/main.yml rename to ansible/roles/opensearch/tasks/main.yml index f7fe0d18..3f1a4667 100644 --- a/ansible/roles/elasticsearch/tasks/main.yml +++ b/ansible/roles/opensearch/tasks/main.yml @@ -1,38 +1,43 @@ - name: Stop and remove container vars: container_names: - - elasticsearch + - opensearch-master + - opensearch-data-ingest1 + - opensearch-data-ingest2 + - opensearch-data-ingest3 + - opensearch-coordinator + - opensearch-dashboards containers: "{{ item }}" include_tasks: file: roles/common/tasks/docker_stop_remove_tasks.yml apply: tags: - - elasticsearch + - opensearch loop: "{{ container_names | batch(1) | list }}" tags: - - elasticsearch + - opensearch - name: Spin up new container docker_compose: - project_src: "temp/compose/elasticsearch" + project_src: "temp/compose/opensearch" build: true debug: true become: true register: started_containers tags: - - elasticsearch + - opensearch -- name: Wait for elasticsearch to start - command: docker exec elasticsearch curl localhost:9200/_recovery -sS +- name: Wait for opensearch to start + command: docker exec opensearch curl localhost:9200/_recovery -sS register: result until: result.stdout == '{}' retries: 30 delay: 5 # Seconds become: true tags: - - elasticsearch - - elasticsearch_base_indexes - - elasticsearch_knn_indexes + - opensearch + - opensearch_base_indexes + - opensearch_knn_indexes - name: Create base indexes shell: @@ -40,7 +45,7 @@ chdir: "temp/minitask" register: create_base tags: - - elasticsearch_base_indexes + - opensearch_base_indexes - debug: var=create_base @@ -50,6 +55,6 @@ chdir: "temp/knn_indexing" register: create_knn tags: - - elasticsearch_knn_indexes + - opensearch_knn_indexes - debug: var=create_knn diff --git a/ansible/site.yml b/ansible/site.yml index 18112b89..e8cec1f9 100644 --- a/ansible/site.yml +++ b/ansible/site.yml @@ -17,7 +17,7 @@ - hosts: targets roles: - - elasticsearch + - opensearch - hosts: targets roles: diff --git a/caddy/Caddyfile b/caddy/Caddyfile new file mode 100644 index 00000000..057947db --- /dev/null +++ b/caddy/Caddyfile @@ -0,0 +1,7 @@ +anu.jkl.io { + reverse_proxy localhost:3002 + route /api/* { + uri strip_prefix /api + reverse_proxy localhost:3001 + } +} diff --git a/caddy/Dockerfile b/caddy/Dockerfile new file mode 100644 index 00000000..200a1aa6 --- /dev/null +++ b/caddy/Dockerfile @@ -0,0 +1,6 @@ +FROM caddy:2-alpine + +COPY Caddyfile /etc/caddy/Caddyfile + +EXPOSE 80 +EXPOSE 443 diff --git a/compose/backend/docker-compose.yml b/compose/backend/docker-compose.yml new file mode 100644 index 00000000..ac1df549 --- /dev/null +++ b/compose/backend/docker-compose.yml @@ -0,0 +1,7 @@ +version: "3.5" +services: + backend: + container_name: backend + network_mode: host + build: + context: ../../ diff --git a/compose/caddy/docker-compose.yml b/compose/caddy/docker-compose.yml new file mode 100644 index 00000000..332d2cc8 --- /dev/null +++ b/compose/caddy/docker-compose.yml @@ -0,0 +1,8 @@ +version: "3.5" +services: + caddy: + container_name: caddy + network_mode: host + build: + context: ../../caddy + restart: unless-stopped diff --git a/compose/frontend/docker-compose.yml b/compose/frontend/docker-compose.yml new file mode 100644 index 00000000..f784a1c6 --- /dev/null +++ b/compose/frontend/docker-compose.yml @@ -0,0 +1,6 @@ +version: "3.5" +services: + new-frontend: + build: + context: ../../frontend_new + network_mode: host diff --git a/compose/opensearch/docker-compose.yml b/compose/opensearch/docker-compose.yml index 304ece93..f9a9a03d 100644 --- a/compose/opensearch/docker-compose.yml +++ b/compose/opensearch/docker-compose.yml @@ -23,9 +23,7 @@ services: hard: 65536 volumes: - opensearch-master:/usr/share/opensearch/data - ports: - - 9200:9200 - - 9600:9600 # required for Performance Analyzer + network_mode: host networks: - opensearch-net opensearch-data-ingest1: @@ -49,9 +47,7 @@ services: nofile: soft: 65536 hard: 65536 - ports: - - 9201:9201 - - 9601:9601 # required for Performance Analyzer + network_mode: host volumes: - opensearch-data-ingest1:/usr/share/opensearch/data networks: @@ -77,9 +73,7 @@ services: nofile: soft: 65536 hard: 65536 - ports: - - 9202:9202 - - 9602:9602 # required for Performance Analyzer + network_mode: host volumes: - opensearch-data-ingest2:/usr/share/opensearch/data networks: @@ -105,9 +99,7 @@ services: nofile: soft: 65536 hard: 65536 - ports: - - 9203:9203 - - 9603:9603 # required for Performance Analyzer + network_mode: host volumes: - opensearch-data-ingest2:/usr/share/opensearch/data networks: @@ -132,9 +124,7 @@ services: nofile: soft: 65536 hard: 65536 - ports: - - 9204:9204 - - 9604:9604 # required for Performance Analyzer + network_mode: host volumes: - opensearch-coordinator:/usr/share/opensearch/data networks: @@ -142,10 +132,7 @@ services: opensearch-dashboards: image: opensearchproject/opensearch-dashboards:latest container_name: opensearch-dashboards - ports: - - 5601:5601 - expose: - - "5601" + network_mode: host environment: OPENSEARCH_HOSTS: '["http://opensearch-master:9200","http://opensearch-master:9201","http://opensearch-master:9202","http://opensearch-master:9203","http://opensearch-master:9204"]' networks: diff --git a/compose/opensearch/opensearch.yml b/compose/opensearch/opensearch.yml index 553b3095..73d26dc6 100644 --- a/compose/opensearch/opensearch.yml +++ b/compose/opensearch/opensearch.yml @@ -7,12 +7,6 @@ network.host: 0.0.0.0 # # set to 1 to allow single node clusters # discovery.zen.minimum_master_nodes: 1 -node.name: "node-1" -# explicitly specify the node will be function as master, data and ingest -node.master: true -node.data: true -node.ingest: true - # Setting network.host to a non-loopback address enables the annoying bootstrap checks. "Single-node" mode disables them again. #discovery.type: single-node diff --git a/compose/redis/docker-compose.yml b/compose/redis/docker-compose.yml index edb9be84..e66de534 100644 --- a/compose/redis/docker-compose.yml +++ b/compose/redis/docker-compose.yml @@ -7,4 +7,3 @@ services: context: ../../modules/CuckooFilter volumes: - ../redis:/redis - command: redis-server /usr/local/etc/redis/redis.conf diff --git a/frontend_new/Dockerfile b/frontend_new/Dockerfile new file mode 100644 index 00000000..5d743098 --- /dev/null +++ b/frontend_new/Dockerfile @@ -0,0 +1,12 @@ +FROM node:lts-alpine + +RUN npm install -g http-server + +WORKDIR /app + +COPY . . +RUN npm install +RUN npm run build + +EXPOSE 3002 +CMD ["http-server", "-p", "3002"]