From d7317e9a27ae2a0433cecb7f3d590791cce12e4a Mon Sep 17 00:00:00 2001 From: Micah Baker Date: Sat, 16 Nov 2024 13:18:47 -0800 Subject: [PATCH] Remove config files and create alembic workflow --- .github/workflows/alembic.yml | 47 +++++++ .github/workflows/pytest_unit.yml | 4 +- config/cron.sh | 1 - config/csss-site.service | 14 --- config/export_secrets.sh | 10 -- config/nginx.conf | 30 ----- config/sudoers.conf | 2 - config/update_config.sh | 28 ----- deploy.sh | 8 -- fresh_setup.sh | 135 --------------------- gunicorn_start.sh => junicorn_start_dev.sh | 12 +- 11 files changed, 55 insertions(+), 236 deletions(-) create mode 100644 .github/workflows/alembic.yml delete mode 100644 config/csss-site.service delete mode 100644 config/export_secrets.sh delete mode 100755 config/nginx.conf delete mode 100644 config/sudoers.conf delete mode 100755 config/update_config.sh delete mode 100755 deploy.sh delete mode 100755 fresh_setup.sh rename gunicorn_start.sh => junicorn_start_dev.sh (63%) diff --git a/.github/workflows/alembic.yml b/.github/workflows/alembic.yml new file mode 100644 index 0000000..23c14cc --- /dev/null +++ b/.github/workflows/alembic.yml @@ -0,0 +1,47 @@ +name: Alembic Upgrade Head +on: pull_request + +jobs: + alembic_upgrade_head: + runs-on: ubuntu-latest + + services: + postgres: + image: postgres:15 + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U runner" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + env: + POSTGRES_USER: runner + POSTGRES_DB: main + POSTGRES_HOST_AUTH_METHOD: trust + + steps: + - uses: actions/checkout@v4 + + - name: Wait for PostgreSQL to be ready + run: | + until pg_isready -h localhost -p 5432 -U runner; do + echo "Waiting for PostgreSQL..." + sleep 1 + done + + - name: Install dependencies + run: | + sudo apt-get install python3.11 python3.11-venv + python3.11 -m pip install --upgrade pip + python3.11 -m venv venv + source ./venv/bin/activate + pip install -r requirements.txt + + # This will fail if there are divergent heads and alembic gets confused; + # e.g., un-sanitarily merging main into a dev branch. + - name: Run alembic upgrade head + run: | + source ./venv/bin/activate + cd src + alembic upgrade head diff --git a/.github/workflows/pytest_unit.yml b/.github/workflows/pytest_unit.yml index d354c0c..b48479d 100644 --- a/.github/workflows/pytest_unit.yml +++ b/.github/workflows/pytest_unit.yml @@ -2,7 +2,7 @@ name: Unit Tests on: pull_request jobs: - test: + unit_tests: runs-on: ubuntu-latest timeout-minutes: 5 @@ -31,4 +31,4 @@ jobs: - name: Run unit tests run: | source ./venv/bin/activate - pytest ./tests/unit -v \ No newline at end of file + pytest ./tests/unit -v diff --git a/config/cron.sh b/config/cron.sh index 0660237..955178b 100644 --- a/config/cron.sh +++ b/config/cron.sh @@ -3,4 +3,3 @@ # run the daily script at 1am every morning # TODO: make sure timezone is PST crontab -l | { cat; echo "0 1 * * * /home/csss-site/csss-site-backend/src/cron/daily.py"; } | crontab - - diff --git a/config/csss-site.service b/config/csss-site.service deleted file mode 100644 index fa3e704..0000000 --- a/config/csss-site.service +++ /dev/null @@ -1,14 +0,0 @@ -[Unit] -Description=CSSS Backend -After=network.target -StartLimitIntervalSec=0 - -[Service] -Type=exec -Restart=always -RestartSec=1 -User=csss-site -ExecStart=/home/csss-site/csss-site-backend/gunicorn_start.sh - -[Install] -WantedBy=multi-user.target diff --git a/config/export_secrets.sh b/config/export_secrets.sh deleted file mode 100644 index 41eab07..0000000 --- a/config/export_secrets.sh +++ /dev/null @@ -1,10 +0,0 @@ -# TODO: only fill out this file in production -export GMAIL_USERNAME = "todo" -export GMAIL_PASSWORD = "todo" -export GOOGLE_DRIVE_TOKEN = "todo" -export GITHUB_TOKEN = "todo" -export DISCORD_TOKEN = "todo" - -export CSSS_GUILD_ID = "todo" -export SFU_API_TOKEN = "todo" - diff --git a/config/nginx.conf b/config/nginx.conf deleted file mode 100755 index bdd1e44..0000000 --- a/config/nginx.conf +++ /dev/null @@ -1,30 +0,0 @@ -upstream backend { - server unix:/var/www/gunicorn.sock fail_timeout=0; -} - -server { - keepalive_timeout 5; - client_max_body_size 1G; # Was 4G - - access_log /var/www/logs/csss-site-backend/nginx-access.log; - error_log /var/www/logs/csss-site-backend/nginx-error.log; - - location / { - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header Host $http_host; - proxy_redirect off; - - if (!-f $request_filename) { - proxy_pass http://backend; - break; - } - - proxy_hide_header Access-Control-Allow-Origin; - proxy_hide_header Access-Control-Allow-Credentials; - add_header Access-Control-Allow-Origin https://new.sfucsss.org always; - add_header Access-Control-Allow-Credentials true; - } - - server_name api.sfucsss.org; - listen 80; -} diff --git a/config/sudoers.conf b/config/sudoers.conf deleted file mode 100644 index 37a44a6..0000000 --- a/config/sudoers.conf +++ /dev/null @@ -1,2 +0,0 @@ -# enable the csss-site user to deploy the website -csss-site ALL=(ALL) NOPASSWD: /usr/bin/systemctl restart nginx, /usr/bin/systemctl restart csss-site diff --git a/config/update_config.sh b/config/update_config.sh deleted file mode 100755 index 53cb462..0000000 --- a/config/update_config.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash - -# make sure user is root -user=$(whoami) -if [ $user != 'root' ]; then - echo "this script must be run as the superuser." - exit 1 -fi - -echo "1. update nginx configs" -cp /home/csss-site/csss-site-backend/config/nginx.conf /etc/nginx/sites-available/csss-site -certbot --nginx # reconfigure the server with SSL certificates -nginx -t -# only restart nginx if config is valid -if [ $? -eq 0 ]; then - systemctl restart nginx -fi - -echo "2. update csss-site service config" -systemd-analyze verify /home/csss-site/csss-site-backend/config/csss-site.service -# only use new service if it is valid -if [ $? -eq 0 ]; then - cp /home/csss-site/csss-site-backend/config/csss-site.service /etc/systemd/system/csss-site.service - systemctl restart csss-site -fi - -echo "3. update sudo config" -cp /home/csss-site/csss-site-backend/config/sudoers.conf /etc/sudoers.d/csss-site diff --git a/deploy.sh b/deploy.sh deleted file mode 100755 index 254379f..0000000 --- a/deploy.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -# NOTE: this script assumes that the local filetree contains what you intend to deploy -# please run the config/update_config.sh if the configuration files are new, or fresh_start.sh if nothing has been installed yet - -echo "restarting nginx and gunicorn, gracefully" -sudo systemctl restart nginx -sudo systemctl restart csss-site diff --git a/fresh_setup.sh b/fresh_setup.sh deleted file mode 100755 index dbde163..0000000 --- a/fresh_setup.sh +++ /dev/null @@ -1,135 +0,0 @@ -#!/bin/bash - -# this is a script for seting up the website from a fresh install - -# TODO: -# - look into `apt install unattended-upgrades` -# - look into activating fail2ban for ssh protection (I doubt we'll need this unless we get too much random traffic) - -# make sure user is root -user=$(whoami) -if [ $user != 'root' ]; then - echo "this script must be run as the superuser." - exit 1 -fi - -echo "hi sysadmin!" -echo "this script will install (almost) everything needed to run the csss website" -echo "(make sure you are running on a Debian 12 Linux machine as the superuser!)" - -# ask the user for consent to proceed -while true; do - echo "(P)roceed, (c)ancel?" - read choice - - if [ $choice = 'P' ]; then - break - elif [ $choice = 'c' ]; then - exit 0 - else - echo "Not sure what you mean..." - fi -done - -echo "----" -echo "configure apt sources..." -echo "deb https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list -wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - - -echo "----" -echo "update and upgrade apt..." -apt update && apt upgrade -y - -echo "----" -echo "install packages..." -apt install git software-properties-common python3.11 python3.11-venv libaugeas0 nginx postgresql-15 postgresql-contrib -y -# install certbot -python3 -m venv /opt/certbot -/opt/certbot/bin/pip install --upgrade pip -/opt/certbot/bin/pip install certbot certbot-nginx -ln -s /opt/certbot/bin/certbot /usr/bin/certbot - -echo "----" -echo "add user csss_site..." -useradd csss-site -m # -m: has home /home/csss-site -usermod -L csss-site # -L: cannot login -chsh -s /usr/bin/bash csss-site # make user csss-site use the bash shell -cd /home/csss-site - -echo "----" -echo "clone repository csss-site-backend..." -sudo -u csss-site git clone https://github.com/CSSS/csss-site-backend csss-site-backend - -echo "----" -echo "configure sudo for csss-site..." -cp csss-site-backend/config/sudoers.conf /etc/sudoers.d/csss-site - -echo "----" -echo "configure nginx..." -# www-data and /var/www stuff -usermod -aG www-data csss-site -mkdir /var/www/logs -mkdir /var/www/logs/csss-site-backend -chown -R www-data:www-data /var/www -chmod -R ug=rwx,o=rx /var/www -# nginx config files -cp csss-site-backend/config/nginx.conf /etc/nginx/sites-available/csss-site-backend -# remove default configuration to prevent funky certbot behaviour -rm /etc/nginx/sites-enabled/default - -# prompt user to modify the nginx configuration if they so please -echo "Do you want to modify the nginx configuration file?" -while true; do - echo "(M)odify, (c)ontinue?" - read choice - - if [ $choice = 'M' ]; then - vim /etc/nginx/sites-available/csss-site-backend - break - elif [ $choice = 'c' ]; then - break - else - echo "Not sure what you mean..." - fi -done - -ln -s /etc/nginx/sites-available/csss-site-backend /etc/nginx/sites-enabled/csss-site-backend -echo "You'll need to fill out the certbot configuration manually." -echo "Use csss-sysadmin@sfu.ca for contact email." -certbot --nginx -nginx -t - -echo "----" -echo "start nginx..." -systemctl start nginx && systemctl enable nginx - -echo "----" -echo "configure postgres..." -# see https://towardsdatascience.com/setting-up-postgresql-in-debian-based-linux-e4985b0b766f for more details -# NOTE: the installation of postgresql-15 creates the postgres user, which has special privileges -sudo -u postgres createdb --no-password main -sudo -u postgres createuser --no-password csss-site -sudo -u postgres psql --command='GRANT ALL PRIVILEGES ON DATABASE main TO "csss-site"' -sudo -u postgres psql main --command='GRANT ALL ON SCHEMA public TO "csss-site"' - -echo "----" -echo "create a virtual environment for csss-site..." -sudo -u csss-site python3.11 -m venv .venv -source .venv/bin/activate - -echo "----" -echo "install pip packages for csss-site..." -cd csss-site-backend -sudo -u csss-site /home/csss-site/.venv/bin/pip install -r requirements.txt -deactivate - -echo "----" -echo "configure csss-site systemd service..." -cp config/csss-site.service /etc/systemd/system/csss-site.service - -echo "----" -echo "start csss-site..." -systemctl start csss-site && systemctl enable csss-site - -echo "----" -echo "all done." diff --git a/gunicorn_start.sh b/junicorn_start_dev.sh similarity index 63% rename from gunicorn_start.sh rename to junicorn_start_dev.sh index 67803c1..942ee55 100755 --- a/gunicorn_start.sh +++ b/junicorn_start_dev.sh @@ -1,19 +1,19 @@ #!/bin/bash NAME=csss-site -DIR=/home/csss-site/csss-site-backend/src -USER=csss-site -GROUP=csss-site +DIR= # fill this out yourself +USER= # fill this out yourself +GROUP= # fill this out yourself WORKERS=2 # TODO: should we increase this? WORKER_CLASS=uvicorn.workers.UvicornWorker -VENV=/home/csss-site/.venv/bin/activate -BIND=unix:/var/www/gunicorn.sock +VENV= # fill this out yourself +BIND= # e.g., unix:/var/www/gunicorn.sock LOG_LEVEL=error cd $DIR source $VENV -exec gunicorn main:app \ +gunicorn main:app \ --name $NAME \ --workers $WORKERS \ --worker-class $WORKER_CLASS \