Migrate from Apache to Gunicorn + Nginx

.github/workflows/django-test.yaml

@@ -28,6 +28,7 @@ jobs:
       EMAIL_HOST_PASSWORD: ${{secrets.EMAIL_HOST_PASSWORD}}
       FROM_EMAIL: ${{secrets.FROM_EMAIL}}
       ALLOWED_HOSTS: ${{secrets.ALLOWED_HOSTS}}
+      POSTGRES_USER: $${{secrets.POSTGRES_USER}}
       POSTGRES_PASS: $${{secrets.POSTGRES_PASS}}
       GIT_TOKEN: ${{secrets.GIT_TOKEN}}
       DEBUG: ${{secrets.DEBUG}}

.gitignore

@@ -4,7 +4,6 @@ post_metaimgs/
 profile_pics/
 uploads/
 django_project/media/django_ckeditor_5/
-backups/
 datadump.json
 db.sqlite3-journal
 
@@ -36,4 +35,8 @@ blogthedata.log
 blogthedata_detailed.log
 
 # migrations
-migration_backup
+migration_backup
+
+# backups
+blogthedata*.sql
+linode*.img

.vscode/launch.json

@@ -24,7 +24,7 @@
     },
     {
       "name": "Django_Start",
-      "python": "/Users/johnsolly/Documents/code/blogthedata/django_project/venv/bin/python3",
+      "python": "/Users/johnsolly/Documents/code/blogthedata/venv/bin/python3",
       "type": "python",
       "request": "launch",
       "program": "/Users/johnsolly/Documents/code/blogthedata/django_project/manage.py",

config/blogthedata.code-workspace → config/blogthedata.code-workspace.code-workspace

@@ -13,7 +13,7 @@
 		"python.formatting.provider": "black",
 		"python.formatting.autopep8Args": ["--ignore","E402"],
 		"python.terminal.activateEnvInCurrentTerminal": true,
-		"python.defaultInterpreterPath": "/Users/johnsolly/Documents/code/blogthedata/django_project/venv/bin/python3",
+		"python.defaultInterpreterPath": "/Users/johnsolly/Documents/code/blogthedata/venv/bin/python3",
 		"python.terminal.activateEnvironment": true,
 		"editor.rulers": [
 			79,
@@ -35,6 +35,7 @@
 		"window.autoDetectColorScheme": true,
 		"terminal.integrated.enableMultiLinePasteWarning": false,
 		"diffEditor.ignoreTrimWhitespace": false,
+		"kite.showWelcomeNotificationOnStartup": false,
 		"conventionalCommits.scopes": [
 			"ckeditor",
 			"blog model",

config/configure_django_new.md

@@ -0,0 +1,86 @@
+- create linode
+- local$ ssh root@<ip_address> (from local)
+
+### Initial Hardening
+```sh
+$ apt update -y
+$ apt dist-upgrade -y
+$ reboot
+$ apt update -y 
+$ apt install unattended-upgrades
+$ dpkg-reconfigure --priority=low unattended-upgrades
+$ useradd -m -s /bin/bash john && passwd john
+$ ls /home # should show new user's folder
+$ cat /etc/passwd # New user should be at the bottom of the file
+$ which sudo # Should output /usr/bin/sudo. If not, run apt install sudo
+$ visudo # See groups
+$ usermod -aG sudo john # Add john to sudo group
+$ groups john # Should show that he is in the sudo group
+$ su - john # Switch to john user
+john$ sudo apt update # Make sure john can run a sudo command
+$ exit # You might need to do this twice to get back to local machine
+local$ ssh-copy-id -i ~/.ssh/id_rsa.pub john@<ip_address>
+local$ ssh john@<ip_address>
+john$ sudo nano /etc/ssh/sshd_config
+# Change PermitRootLogin to <no>
+# Uncomment PasswordAuthentication and change to no
+# Add <AllowUsers john> bellow MaxSessions
+$ sudo systemctl restart sshd
+$ sudo service ssh status # Make sure it is running
+local$ ssh john@<ip_address> # make sure you can still ssh in, in a new tab
+john$ ss -atpu # Observe open ports to make sure it's all good!
+$ hostnamectl set-hostname django-server
+$ nano /etc/hosts # add a line of <<ip_address> blogthedata.com django-server>
+# TODO: Add firewall rules
+```
+
+### Configuring App
+```sh
+$ sudo apt update -y
+$ sudo apt-get install python3-venv -y
+# Follow tutorial to get ssh keys on the server
+# https://docs.github.com/en/authentication/connecting-to-github-with-ssh/generating-a-new-ssh-key-and-adding-it-to-the-ssh-agent
+$ git clone git@github.com:jsolly/blogthedata.git
+$ python3 -m venv ~/venv
+$ source ~/venv/bin/activate
+$ python3 -m pip install --upgrade pip
+$ python3 -m pip install wheel
+$ python3 -m pip install -r ~/blogthedata/django_project/requirements/requirements.txt
+$ nano ~/blogthedata/.env
+# copy over parameters from blogthedata/sample.env and set them as needed
+$ sudo ufw allow 8000 # We will turn this off later. It's just for testing.
+$ python3 ~/blogthedata/django_project/manage.py runserver 0.0.0.0:8000
+# in a browser navigate to <ip_address>:8000
+# App should be loading
+```
+
+### Install dependencies
+```bash
+$ sudo apt-get install python3-pip python3-dev libpq-dev postgresql postgresql-contrib nginx
+$ sudo -u postgres psql
+postgres=# CREATE DATABASE blogthedata;
+postgres=# CREATE USER blogthedatauser WITH PASSWORD 'password';
+postgres=# ALTER ROLE blogthedatauser SET client_encoding TO 'utf8';
+postgres=# ALTER ROLE blogthedatauser SET default_transaction_isolation TO 'read committed';
+postgres=# ALTER ROLE blogthedatauser SET timezone TO 'UTC';
+postgres=# GRANT ALL PRIVILEGES ON DATABASE blogthedata TO blogthedatauser;
+```
+### A note on an existing database being imported. You may need to run these commands:
+```
+$ sudo su postgres
+$ psql blogthedata -c "GRANT ALL ON ALL TABLES IN SCHEMA public to blogthedatauser;"
+$ psql blogthedata -c "GRANT ALL ON ALL SEQUENCES IN SCHEMA public to blogthedatauser;"
+$ psql blogthedata -c "GRANT ALL ON ALL FUNCTIONS IN SCHEMA public to blogthedatauser;"
+```
+
+### Restore existing DB (If needed)
+```
+# Copy db into server
+postgres=# exit
+$ sudo su postgres
+$ psql blogthedata < ~/blogthedata/backups/blogthedata_db_6_20_22.sql
+```
+### Continue working on Gunicorn and Nginx
+```
+# Follow a guide on Gunicorn/Nginx 
+$ sudo ufw delete allow 8000

django_project/blog/forms.py

@@ -8,7 +8,7 @@
 choices = Category.objects.all().values_list(
     "name", "name"
 )
-# comment this if doing an initial DB migration or changing databases
+# comment this if doing an initial DB migration or changing databases.
 
 
 class PostForm(forms.ModelForm):

django_project/blog/templates/blog/parts/about_me.html

@@ -4,7 +4,7 @@ <h3>About Me</h3>
   <div class="row no-gutters">
     <div class="col-md-2">
       <img src="{% static 'jsolly.jpeg' %}" class="card-img"
-        alt="{{ my_profile.user.first_name }} {{ my_profile.user.last_name }} Profile Picture">
+        alt="John Solly Profile Picture">
     </div>
     <div class="col-md-10">
       <div class="card-body">

django_project/blog/views.py

@@ -13,9 +13,9 @@
     UpdateView,
     DeleteView,
 )
-from users.models import Profile
 import aiohttp
 import asyncio
+import ssl
 
 
 class HomeView(ListView):
@@ -29,12 +29,6 @@ def get_queryset(self):
             return Post.objects.all()
         return Post.objects.active()
 
-    def get_context_data(self, *args, **kwargs):  # Use a Context processor?
-        context = super().get_context_data(*args, **kwargs)
-        my_user = User.objects.get(username="John_Solly")
-        context["my_profile"] = Profile.objects.get(user=my_user)
-        return context
-
 
 class UserPostListView(ListView):  # Not actively worked on
     model = Post
@@ -135,7 +129,7 @@ def road_map_view(request):
     HEADERS = {"Authorization": f"token {GIT_TOKEN}"}
 
     async def make_request(session, url, params=None):
-        async with session.get(url, params=params) as resp:
+        async with session.get(url, params=params, ssl=ssl.SSLContext()) as resp:
             return await resp.json()
 
     async def main(urls):
@@ -180,7 +174,7 @@ async def main(urls):
         issue for issue in all_open_issues if issue["url"] in next_sprint_issue_urls
     ]
 
-    sprint_number = date.today().isocalendar().week // 2  # Two week sprints
+    sprint_number = date.today().isocalendar()[1] // 2  # Two week sprints
     return render(
         request,
         "blog/roadmap.html",

django_project/django_project/settings.py

@@ -8,6 +8,7 @@
 """
 import os
 from dotenv import load_dotenv
+import psycopg2
 
 load_dotenv()
 
@@ -24,15 +25,15 @@
 DEBUG = False
 CAPTCHA_TEST_MODE = False
 
-# HTTPS SETTINGS
+# # HTTPS SETTINGS
 SESSION_COOKIE_SECURE = True
 SESSION_COOKIE_HTTPONLY = True
 CSRF_COOKIE_SECURE = True
 SECURE_SSL_REDIRECT = True
 SECURE_BROWSER_XSS_FILTER = True
 SECURE_CONTENT_TYPE_NOSNIFF = True
 
-# HSTS SETTINGS
+# # HSTS SETTINGS
 SECURE_HSTS_SECONDS = 31557600
 SECURE_HSTS_PRELOAD = True
 SECURE_HSTS_INCLUDE_SUBDOMAINS = True
@@ -73,6 +74,7 @@
     # HTTPS SETTINGS
     SESSION_COOKIE_SECURE = False
     SESSION_COOKIE_HTTPONLY = False
+    SESSION_COOKIE_SECURE = False
     SECURE_SSL_REDIRECT = False
 
 
@@ -207,14 +209,28 @@
     "default": {
         "ENGINE": "django.db.backends.postgresql_psycopg2",
         "NAME": "blogthedata",
-        "USER": "postgres",
+        "USER": os.environ["POSTGRES_USER"],
         "PASSWORD": os.environ["POSTGRES_PASS"],
         "HOST": "localhost",
         "PORT": "5432",
+        "OPTIONS": {
+            "isolation_level": psycopg2.extensions.ISOLATION_LEVEL_SERIALIZABLE,
+        },
     }
 }
+
+# DATABASES = {
+#     "default": {
+#         "ENGINE": "django.db.backends.sqlite3",
+#         "NAME": os.path.join(BASE_DIR, "db.sqlite3"),
+#     }
+# }
 import sys
-if len({item for item in ["testFile", "discover"] if any(item in arg for arg in sys.argv)}) > 0:
+
+found_count = len(
+    {item for item in ["testFile", "discover"] if any(item in arg for arg in sys.argv)}
+)
+if found_count > 0:
     DATABASES = {
         "default": {
             "ENGINE": "django.db.backends.sqlite3",

django_project/requirements/requirements.txt

@@ -15,8 +15,9 @@ django-simple-captcha==0.5.17
 django-sri==0.3.0
 filetype==1.0.13
 flake8==4.0.1
+gunicorn==20.1.0
 Pillow==9.1.1
-psycopg2==2.9.3
+psycopg2-binary==2.9.3
 python-dotenv==0.20.0
 requests==2.28.0
 selenium==4.2.0

django_project/requirements/requirements_6_8_22.txt → django_project/requirements/requirements_6_19.txt

@@ -1,23 +1,23 @@
 aiohttp==3.8.1
 black==22.3.0
+Brotli==1.0.9
 chromedriver-autoinstaller==0.3.1
-coverage==6.3.2
-Django==3.2.13
-django-admin-honeypot==1.1.0
-django-ckeditor-5==0.1.3
-django-coverage-plugin==2.0.2
+coverage==6.4.1
+Django==4.0.5
+django-ckeditor==6.4.2
+django-ckeditor-5==0.1.5
+django-coverage-plugin==2.0.3
 django-csp==3.7
 django-fastdev==1.7.2
 django-livereload-server==0.4
-django-non-dark-admin==2.0.2
 django-robots==5.0
-django-simple-captcha==0.5.14
+django-simple-captcha==0.5.17
 django-sri==0.3.0
 filetype==1.0.13
 flake8==4.0.1
-geckodriver-autoinstaller==0.1.0
 Pillow==9.1.1
 psycopg2==2.9.3
 python-dotenv==0.20.0
-requests==2.25.1
-selenium==4.1.3
+requests==2.28.0
+selenium==4.2.0
+whitenoise==6.2.0

django_project/tests/test_functional_ui.py

@@ -17,15 +17,20 @@
 setup()
 from users.models import User
 from blog.models import Category, Post
+
 # from unittest import skip
 
 # geckodriver_autoinstaller.install()
+import ssl
+if not os.environ.get("PYTHONHTTPSVERIFY", "") and getattr(
+    ssl, "_create_unverified_context", None
+):
+    ssl._create_default_https_context = ssl._create_unverified_context
 chromedriver_autoinstaller.install()
 
 
 # @skip("Tests take too long to run")
 class TestFunctionalUI(StaticLiveServerTestCase):
-
     def setUp(self):
         self.general_password = "T3stingIsFun!"
 
@@ -75,7 +80,7 @@ def create_user(provided_username, super_user=False):
                 snippet="Long ago, the four nations lived together in harmony.",
                 content="Long ago, the four nations lived together in harmony. Then everything changed when the fire nation attacked.",
                 # date_posted = ""
-                author=self.super_user
+                author=self.super_user,
             )
 
         # URLs
@@ -107,9 +112,9 @@ def test_author_post_crud(self):
             "Super User's Post"
         )
         self.browser.find_element(by=By.NAME, value="slug").send_keys("super-user-post")
-        Select(self.browser.find_element(by=By.NAME, value="category")).select_by_visible_text(
-            "Productivity"
-        )
+        Select(
+            self.browser.find_element(by=By.NAME, value="category")
+        ).select_by_visible_text("Productivity")
         actions = ActionChains(self.browser)
         actions.send_keys(Keys.TAB * 4).perform()
         actions.send_keys("Some Content").perform()
@@ -174,12 +179,8 @@ def test_anonymous_can_register_workflow(self):
         self.browser.find_element(by=By.NAME, value="username").send_keys(
             "selenium_user"
         )
-        self.browser.find_element(by=By.NAME, value="first_name").send_keys(
-            "Michael"
-        )
-        self.browser.find_element(by=By.NAME, value="last_name").send_keys(
-            "Jenkins"
-        )
+        self.browser.find_element(by=By.NAME, value="first_name").send_keys("Michael")
+        self.browser.find_element(by=By.NAME, value="last_name").send_keys("Jenkins")
         self.browser.find_element(by=By.NAME, value="email").send_keys(
             "selenium_user@invalid.com"
         )

django_project/tests/test_views.py

@@ -3,7 +3,6 @@
 from blog.models import Post
 from blog.forms import PostForm
 from users.forms import UserRegisterForm, UserUpdateForm, ProfileUpdateForm
-from users.models import Profile
 
 
 class TestViews(SetUp):
@@ -20,7 +19,6 @@ def test_home_view(self):  # TODO add check for draft post
         response = self.client.get(reverse("blog-home"))
         self.assertEqual(response.status_code, 200)
         self.assertTemplateUsed(response, "blog/home.html")
-        self.assertIsInstance(response.context["my_profile"], Profile)
         self.assertIsInstance(response.context["posts"][0], Post)
         # self.assertIsInstance(response.context["form"])
 

sample.env

@@ -4,6 +4,7 @@ EMAIL_HOST_USER=""
 EMAIL_HOST_PASSWORD=""
 FROM_EMAIL=""
 ALLOWED_HOSTS="127.0.0.1 .ngrok.io localhost"
+POSTGRES_USER=""
 POSTGRES_PASS=""
 GIT_TOKEN=""
 DEBUG="True"