From d24eb089be7638e9ae7835496c4c0b25e5bbeccc Mon Sep 17 00:00:00 2001 From: Gary O'Neall Date: Sun, 21 Jan 2024 09:52:08 -0800 Subject: [PATCH] Updates for deployment Update the version in the docker-compose.prod file Update the version of the base image to resolve vulnerabilities Update the version of license checker to resolve vulnerabilities Update production dockerfile to automatically migrate and populate the license database Update the production documentation to clarify deployment procedures Signed-off-by: Gary O'Neall --- README-PRODUCTION.md | 38 +++++++++++--------------------------- docker-compose.prod.yml | 2 +- dockerfile.dev | 2 +- dockerfile.prod | 10 ++++++---- requirements.txt | 2 +- 5 files changed, 20 insertions(+), 34 deletions(-) diff --git a/README-PRODUCTION.md b/README-PRODUCTION.md index b1e49633..143b2f54 100644 --- a/README-PRODUCTION.md +++ b/README-PRODUCTION.md @@ -53,21 +53,13 @@ Following are the steps for updating the images: - Build the image by running `docker-compose -f docker-compose.prod.yml build` - Test the image for vulnerability by running `docker scan [image]` where `[image]` is the image name from the docker-compose.prod.yml file - Update any dependencies as needed based on the vulnerability report -- Push the image to AWS ECR - This is not necessary if the image is rebuilt on the deployment architecture - - Login to ECR using the AWS CLI by running `aws ecr get-login-password --region | docker login --username AWS --password-stdin .dkr.ecr..amazonaws.com` replacing the `region` and `account ID` - - Push the images by running `docker-compose -f docker-compose.prod.yml push` - Deploy the images on EC2 - - Clone this repo on the EC2 instance - a convenient way to copy of the docker-compose files - - Login to ECR using the AWS CLI by running `sudo docker login -u AWS -p $(aws ecr get-login-password --region ) .dkr.ecr..amazonaws.com` replacing the `region` and `account ID` - - If the image was built as the same architecture as the production architecture, pull the online-tools image by running `docker pull .dkr.ecr..amazonaws.com/spdx/online-tools:` replacing the , , and - - If the architecture of the deployment architecture is different from the development architecture (e.g. an ARM architecture as is currently configured), the image needs to be built on the deployment machine: + - Clone or update this repo on the EC2 instance - a convenient way to copy of the docker-compose files + - Build the image on the deployment machine: - execute `docker-compose -f docker-compose.prod.yml build` + - Login to ECR using the AWS CLI by running aws ecr get-login-password --region | docker login --username AWS --password-stdin .dkr.ecr..amazonaws.com replacing the region and account ID + - Push the images by running docker-compose -f docker-compose.prod.yml push - Launch the containers with the command `docker-compose -f docker-compose.prod.yml up -d` - - If needed upgrade the database: - - Stop the current Python service with the command `docker exec spdx_prod supervisorctl stop spdx` - - Make migrations by running the command `docker exec spdx_prod python src/manage.py makemigrations` - - Upgrade the database with the command `docker exec spdx_prod python src/manage.py migrate` - - Start the spdx_prod service with the command `docker exec spdx_prod supervisorctl start spdx` # Clean Intialial Install @@ -101,16 +93,17 @@ Following are the steps for a clean initial installaction of the application: - replace `` with the AWS region - replace `` with the specific version of the spdx-online-tools-build to be deployed - Build the image by running `docker-compose -f docker-compose.prod.yml build` -- Push the image to AWS ECR - - Login to ECR using the AWS CLI by running `aws ecr get-login-password --region | docker login --username AWS --password-stdin .dkr.ecr..amazonaws.com` replacing the `region` and `account ID` - - Push the images by running `docker-compose -f docker-compose.prod.yml push` +- Test the image for vulnerability by running `docker scan [image]` where `[image]` is the image name from the docker-compose.prod.yml file + - Update any dependencies as needed based on the vulnerability report - Setup the SSL Certificates - Edit the file scripts/init-letsencrypt.sh replacing the email address and setting staging to 1 if testing, 0 if in production - Execut the scriptrun `chmod +x init-letsencrypt.sh` and `sudo ./init-letsencrypt.sh`. - Deploy the images on EC2 - - Clone this repo on the EC2 instance - a convenient way to copy of the docker-compose files - - Login to ECR using the AWS CLI by running `aws ecr get-login-password --region | docker login --username AWS --password-stdin .dkr.ecr..amazonaws.com` replacing the `region` and `account ID` - - Pull the online-tools image by running `docker pull .dkr.ecr..amazonaws.com/spdx/online-tools:` replacing the , , and + - Clone or update this repo on the EC2 instance - a convenient way to copy of the docker-compose files + - Build the image on the deployment machine: + - execute `docker-compose -f docker-compose.prod.yml build` + - Login to ECR using the AWS CLI by running aws ecr get-login-password --region | docker login --username AWS --password-stdin .dkr.ecr..amazonaws.com replacing the region and account ID + - Push the images by running docker-compose -f docker-compose.prod.yml push - Create the spdx-prod.env file ``` @@ -129,15 +122,6 @@ SQL_PORT=5432 ``` - Launch the containers with the command `docker-compose -f docker-compose.prod.yml up -d` -- Initialize the database - - Find the container ID for the spdx-online-tools by executing `docker ps` - - Open a shell in the spdx-online-tools container by executing `docker exec -it [spdx-online-tools-container-id] /bin/shell` - - Initialize the database using DJango by running `python manage.py migrate` - - Populate the license list database by running `python src/populate.py` -- Restart the spdx - - run supervisorctl `supervisorctl` - - restart spdx `restart spdx` - - exit supervisorctl `exit` # Credits diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 2f484461..8010324c 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -31,7 +31,7 @@ services: build: context: ./ dockerfile: dockerfile.prod - image: 410487266669.dkr.ecr.us-west-2.amazonaws.com/spdx/online-tools:1.2.2 + image: 410487266669.dkr.ecr.us-west-2.amazonaws.com/spdx/online-tools:1.2.3 expose: - 8000 volumes: diff --git a/dockerfile.dev b/dockerfile.dev index 3abafbb6..aa17616b 100644 --- a/dockerfile.dev +++ b/dockerfile.dev @@ -1,5 +1,5 @@ # Use the official python 3 base image -FROM python:3.10.6 as python-container +FROM python:3.10.13 as python-container # Copy the requirements.txt file into the container COPY requirements.txt . diff --git a/dockerfile.prod b/dockerfile.prod index 32e464ec..2d32bf6d 100644 --- a/dockerfile.prod +++ b/dockerfile.prod @@ -1,5 +1,5 @@ -# Using python3.10.6 as the base image -FROM python:3.10.6 as base +# Using python3.10.13 as the base image +FROM python:3.10.13 as base # Create the folder spdx and cd to it WORKDIR /spdx @@ -9,7 +9,7 @@ COPY requirements.txt . # install psycopg2 dependencies RUN apt-get update \ - && apt-get -y install libpq-dev gcc python-dev musl-dev + && apt-get -y install libpq-dev gcc python-dev-is-python3 musl-dev # install psycopg2 RUN pip install psycopg2-binary==2.9.3 @@ -58,5 +58,7 @@ EXPOSE 8000 RUN mv ./supervisor_api.conf /etc/supervisor/conf.d # Start the supervisor -CMD python src/manage.py collectstatic --noinput && \ +CMD python src/manage.py migrate && \ + python src/manage.py collectstatic --noinput && \ + python src/populate.py && \ /usr/bin/supervisord -n -c /etc/supervisor/supervisord.conf diff --git a/requirements.txt b/requirements.txt index 868e5158..752d0631 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,4 +17,4 @@ django-oauth-toolkit==1.5.0 django-rest-framework-social-oauth2==1.1.0 spdx-tools==0.8.2 ntia-conformance-checker==1.1.0 --e git+https://github.com/spdx/spdx-license-matcher.git@v2.5#egg=spdx-license-matcher +-e git+https://github.com/spdx/spdx-license-matcher.git@v2.6#egg=spdx-license-matcher