Merge staging into production - 2024-Jan-12 edition

.github/workflows/broken-link-checker.yaml

@@ -0,0 +1,44 @@
+name: Link checker for articles and flatpages on CantusDB
+
+on:
+  schedule:
+    - cron: "8 8 * * 0" # Cron job will run at 08h08 UTC time every Sunday
+
+jobs:
+  get-all-links:
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+    steps:
+    - id: set-matrix
+      run: |
+        BASE_URL="https://cantusdatabase.org"
+        flatpages=$(curl "$BASE_URL/flatpages-list/" | awk '{ gsub (" ", "\",\"", $0); print}')
+        articles=$(curl "$BASE_URL/articles-list/" | awk '{ gsub (" ", "\",\"", $0); print}')
+        list="{\"links\": [\"${flatpages}\",\"${articles}\"]}"
+        echo $list
+        echo "matrix=$list" >> $GITHUB_OUTPUT
+  link-Checker:
+    runs-on: ubuntu-latest
+    needs: get-all-links
+    strategy:
+      fail-fast: false
+      max-parallel: 4
+      matrix:  ${{fromJson(needs.get-all-links.outputs.matrix)}}
+    steps:
+      - uses: actions/checkout@v3
+      - name: Link Checker
+        id: lychee
+        uses: lycheeverse/lychee-action@v1.8.0
+        with:
+          args: --exclude http:\/\/cantus\.sk.* ${{ matrix.links }}
+          format: json
+          output: /tmp/link-checker-output.txt
+      - name: Curating Link Checker Output
+        run: |
+          echo "***Python Version***"
+          python --version
+          echo "***Invoking Parsing Script***"
+          python "$GITHUB_WORKSPACE/scripts/parse_link_checker_output.py" >> $GITHUB_STEP_SUMMARY
+          echo "***Printing Summary***"
+          cat $GITHUB_STEP_SUMMARY

.gitignore

@@ -104,12 +104,13 @@ django/media/
 .env
 venv
 
-# Fixtures
-django/cantusdb_project/main_app/fixtures
-
-#
+# Configuration files
 config/envs/dev_env
 .devcontainer/
+certificates/
+docker-compose.yml
+config/nginx/conf.d/cantusdb.conf
+cron/cron.txt
 
 # Drupal scripts from old Cantus
 django/cantusdb_project/Drupal_scripts
@@ -119,3 +120,6 @@ django/cantusdb_project/Drupal_scripts
 
 # in case you're working on importing data from a database dump
 *.sql
+
+# cached files for APIs
+api_cache/

config/nginx/conf.d/cantusdb.conf.development

@@ -0,0 +1,80 @@
+# This file is configured for deployment of the CantusDB project in local development.
+# When building the project locally, replace the contents of cantusdb.conf with the
+# contents of this file.
+
+# server {
+#     listen 80;
+# 
+#     server_tokens off;
+# 
+#     location ^~ /.well-known/acme-challenge/ {
+#         root /var/www/lego;
+#     }
+# 
+#     location / {
+#         return 301 https://$host$request_uri;
+#     }
+# }
+
+# server {
+#     # Redirect all https traffic for mass.cantusdatabase.org to cantusdatabase.org
+#     listen 443 ssl;
+# 
+#     server_name mass.cantusdatabase.org;
+# 
+#     ssl_certificate /etc/nginx/ssl/live/certificates/cantusdatabase.org.crt;
+#     ssl_certificate_key /etc/nginx/ssl/live/certificates/cantusdatabase.org.key;
+# 
+#     location / {
+#         return 301 https://cantusdatabase.org$request_uri;
+#     }
+# }
+
+server {
+    # listen 443 default_server http2 ssl;
+    listen 80;
+
+    # ssl_certificate /etc/nginx/ssl/live/certificates/cantusdatabase.org.crt;
+    # ssl_certificate_key /etc/nginx/ssl/live/certificates/cantusdatabase.org.key;
+
+    location / {
+        proxy_pass http://django:8000;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header Host $host;
+        proxy_redirect off;
+    }
+
+    location /static {
+        alias /resources/static/;
+    }
+    location /media {
+        alias /resources/media/;
+    }
+
+    location /concordances {
+        alias /resources/api_cache/concordances.json;
+        expires modified +24h;
+    }
+
+    location = /style.css {
+        root /;
+    }
+    location = /background.jpg {
+        root /;
+    }
+    location = /CantusLogoSmall.gif {
+        root /;
+    }
+    location = /favicon.ico {
+        root /;
+    }
+
+    error_page 502 /502.html;
+    location = /502.html {
+        root /;
+    }
+    error_page 504 /504.html;
+    location = /504.html {
+        root /;
+    }
+}

config/nginx/conf.d/cantusdb.conf → config/nginx/conf.d/cantusdb.conf.production

@@ -4,20 +4,34 @@ server {
     server_tokens off;
 
     location ^~ /.well-known/acme-challenge/ {
-        root /var/www/certbot;
+        root /var/www/lego;
     }
 
     location / {
         return 301 https://$host$request_uri;
     }
 }
 
+server {
+    # Redirect all https traffic for mass.cantusdatabase.org to cantusdatabase.org
+    listen 443 ssl;
+
+    server_name mass.cantusdatabase.org;
+
+    ssl_certificate /etc/nginx/ssl/live/certificates/cantusdatabase.org.crt;
+    ssl_certificate_key /etc/nginx/ssl/live/certificates/cantusdatabase.org.key;
+
+    location / {
+        return 301 https://cantusdatabase.org$request_uri;
+    }
+}
+
 server {
 
     listen 443 default_server http2 ssl;
 
-    ssl_certificate /etc/nginx/ssl/live/cantusdatabase.org/fullchain.pem;
-    ssl_certificate_key /etc/nginx/ssl/live/cantusdatabase.org/privkey.pem;
+    ssl_certificate /etc/nginx/ssl/live/certificates/cantusdatabase.org.crt;
+    ssl_certificate_key /etc/nginx/ssl/live/certificates/cantusdatabase.org.key;
 
     location / {
         proxy_pass http://django:8000;
@@ -32,7 +46,12 @@ server {
     location /media {
         alias /resources/media/;
     }
-
+
+    location /concordances {
+        alias /resources/api_cache/concordances.json;
+        expires modified +24h;
+    }
+
     location = /style.css {
         root /;
     }

config/nginx/conf.d/cantusdb.conf.staging

@@ -0,0 +1,83 @@
+server {
+    listen 80;
+
+    server_tokens off;
+
+    location ^~ /.well-known/acme-challenge/ {
+        root /var/www/lego;
+    }
+
+    location / {
+        return 301 https://$host$request_uri;
+    }
+}
+
+server {
+    # Redirect all https traffic for staging-alias.cantusdatabase.org to staging.cantusdatabase.org
+    # (on Production, this redirects from mass.cantusdatabase.org to cantusdatabase.org.
+    # The Staging server's subdomain "staging-alias" is analogous to Production's "mass" subdomain.)
+    listen 443 ssl;
+
+    # server_name mass.cantusdatabase.org;
+    server_name staging-alias.cantusdatabase.org;
+
+    # ssl_certificate /etc/nginx/ssl/live/certificates/cantusdatabase.org.crt;
+    # ssl_certificate_key /etc/nginx/ssl/live/certificates/cantusdatabase.org.key;
+    ssl_certificate /etc/nginx/ssl/live/certificates/staging.cantusdatabase.org.crt;
+    ssl_certificate_key /etc/nginx/ssl/live/certificates/staging.cantusdatabase.org.key;
+
+    location / {
+        return 301 https://staging.cantusdatabase.org$request_uri;
+    }
+}
+
+server {
+
+    listen 443 default_server http2 ssl;
+
+    # ssl_certificate /etc/nginx/ssl/live/certificates/cantusdatabase.org.crt;
+    # ssl_certificate_key /etc/nginx/ssl/live/certificates/cantusdatabase.org.key;
+    ssl_certificate /etc/nginx/ssl/live/certificates/staging.cantusdatabase.org.crt;
+    ssl_certificate_key /etc/nginx/ssl/live/certificates/staging.cantusdatabase.org.key;
+
+    location / {
+        proxy_pass http://django:8000;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header Host $host;
+        proxy_redirect off;
+    }
+
+    location /static {
+        alias /resources/static/;
+    }
+    location /media {
+        alias /resources/media/;
+    }
+
+    location /concordances {
+        alias /resources/api_cache/concordances.json;
+        expires modified +24h;
+    }
+
+    location = /style.css {
+        root /;
+    }
+    location = /background.jpg {
+        root /;
+    }
+    location = /CantusLogoSmall.gif {
+        root /;
+    }
+    location = /favicon.ico {
+        root /;
+    }
+
+    error_page 502 /502.html;
+    location = /502.html {
+        root /;
+    }
+    error_page 504 /504.html;
+    location = /504.html {
+        root /;
+    }
+}

cron/cron.txt.production

@@ -0,0 +1,11 @@
+######################
+# CantusDB Cron Jobs #
+######################
+
+# Note: This is set up to run on the production and staging servers. If you want to run it on your local machine, you will need to change the paths.
+
+# min   hour    day     month   weekday command
+0       4       *       *       *       bash /home/ubuntu/code/CantusDB/cron/postgres/db_backup.sh
+10      4       *       *       *       bash /home/ubuntu/code/CantusDB/cron/management/manage.sh update_cached_concordances
+40      4       1       *       *       bash /home/ubuntu/code/CantusDB/cron/management/manage.sh populate_next_chant_fields; bash /home/ubuntu/code/CantusDB/cron/management/manage.sh populate_is_last_chant_in_feast
+50      4       *       *       7       /usr/local/bin/docker-compose -f /home/ubuntu/code/CantusDB/docker-compose.yml exec -T nginx lego --path /etc/nginx/ssl/live -d cantusdatabase.org -d www.cantusdatabase.org -d mass.cantusdatabase.org -m UPDATE.ME@example.com --http --http.webroot /var/www/lego/ renew --days 45 --renew-hook "nginx -s reload" 

cron/cron.txt.staging

@@ -0,0 +1,11 @@
+######################
+# CantusDB Cron Jobs #
+######################
+
+# Note: This is set up to run on the production server. If you want to run it on your local machine, you will need to change the paths.
+
+# min   hour    day     month   weekday command
+0       4       *       *       *       bash /home/ubuntu/code/CantusDB/cron/postgres/db_backup.sh
+10      4       *       *       *       bash /home/ubuntu/code/CantusDB/cron/management/manage.sh update_cached_concordances
+40      4       1       *       *       bash /home/ubuntu/code/CantusDB/cron/management/manage.sh populate_next_chant_fields; bash /home/ubuntu/code/CantusDB/cron/management/manage.sh populate_is_last_chant_in_feast
+50      4       *       *       7       /usr/local/bin/docker-compose -f /home/ubuntu/code/CantusDB/docker-compose.yml exec -T nginx lego --path /etc/nginx/ssl/live -d staging.cantusdatabase.org -d staging-alias.cantusdatabase.org -m UPDATE.ME@example.com --http --http.webroot /var/www/lego/ renew --days 45 --renew-hook "nginx -s reload" 

cron/management/manage.sh

@@ -7,4 +7,4 @@
 DOCKER_COMPOSE_FILE=/home/ubuntu/code/CantusDB/docker-compose.yml   # This is the path to the docker-compose file.
 COMMAND=$1                                                          # This is the command to execute.
 
-/usr/local/bin/docker-compose -f $DOCKER_COMPOSE_FILE exec django python manage.py $COMMAND
+/usr/local/bin/docker-compose -f $DOCKER_COMPOSE_FILE exec -T django python manage.py $COMMAND