Backend maintenance and update readme

.gitignore

@@ -26,3 +26,5 @@ backup/*
 
 # IDE specific
 .vscode/
+
+venv

README.md

@@ -68,8 +68,9 @@ docker-compose run --rm backend ./migrate.sh
 
 We use [pip-tools](https://github.com/jazzband/pip-tools) to manage Python dependencies. To change the dependencies:
 
-1. Edit [requirements.in](./backend/docker/requirements.in) to add, remove, or edit a dependency. You only need to put primary dependencies here, that is, the ones explicitly needed by our source code. pip-tools will take care of adding their dependencies.
+1. Edit [requirements.in](./backend/docker/requirements.in) file to add, remove, or update dependencies as needed. Include only the primary dependencies—those directly required by our source code—in this file. pip-tools will automatically manage and incorporate any transitive dependencies. For routine maintenance, you may specify both primary and transitive dependencies with pinned versions to ensure consistent updates.
 1. Run `docker-compose run --rm backend pip-compile docker/requirements.in` to generate a new [requirements.txt](./backend/docker/requirements.txt). Note that pip-tools uses the existing `requirements.txt` file when building a new one, so that it can maintain existing versions. To upgrade a package to the newest version compatible with the other libraries, just remove it from the existing `requirements.txt` before running pip-compile.
+    - If you encounter an error while running with docker, a possible workaround is to create a virtual environment within `backend/docker`, install `pip-tools` within that environment, and then run `pip-compile requirements.in` directly from there. This method bypasses network issues that might occur within Docker.
 1. Run `docker-compose build backend` to install the updated requirements into the Docker image.
 
 ### Custom Environment Variables

backend/django/core/utils/utils_external_db.py

@@ -208,9 +208,9 @@ def export_table(project_pk, response):
                         external_db.export_schema,
                         external_db.export_table_name,
                     ):
-                        response[
-                            "success_message"
-                        ] = "Table exists. Dropping and replacing with new output data."
+                        response["success_message"] = (
+                            "Table exists. Dropping and replacing with new output data."
+                        )
 
                         # drop the table and then replace the data
                         data.to_sql(

backend/django/core/utils/utils_form.py

@@ -1,5 +1,4 @@
 import numpy as np
-import pandas as pd
 from django.core.exceptions import ValidationError
 
 from core.utils.util import md5_hash

backend/django/core/views/api_annotate.py

@@ -220,9 +220,9 @@ def verify_label(request, data_pk):
     response = {}
     # if the coder has been un-assigned from the data
     if not DataLabel.objects.filter(data=data).exists():
-        response[
-            "error"
-        ] = "ERROR: This data has no label to verify. Something must have gone wrong."
+        response["error"] = (
+            "ERROR: This data has no label to verify. Something must have gone wrong."
+        )
         return Response(response)
     elif DataLabel.objects.filter(data=data).count() > 1:
         response["error"] = (
@@ -1089,9 +1089,9 @@ def get_label_history(request, project_pk):
     if len(irr_data_df) > 0:
         irr_data_df["edit"] = "no"
         irr_data_df["label"] = irr_data_df["labelID"].apply(lambda x: label_dict[x])
-        irr_data_df[
-            "verified"
-        ] = "N/A (IRR)"  # Technically resolved IRR is verified but perhaps not this user's specific label so just NA
+        irr_data_df["verified"] = (
+            "N/A (IRR)"  # Technically resolved IRR is verified but perhaps not this user's specific label so just NA
+        )
         irr_data_df["verified_by"] = None
         irr_data_df["pre_loaded"] = "No"  # IRR only looks at unlabeled data
 

backend/django/run_tests.sh

@@ -20,5 +20,5 @@ else
   echo "Flake8 Good. Running Tests..."
 fi
 
-coverage run -m py.test "$@"
+coverage run -m pytest "$@"
 coverage report -m --skip-covered

backend/django/smart/settings.py

@@ -8,6 +8,7 @@
 For the full list of settings and their values, see
 https://docs.djangoproject.com/en/1.11/ref/settings/
 """
+
 import os
 
 import redis

backend/django/smart/urls.py

@@ -13,6 +13,7 @@
     1. Import the include() function: from django.urls import re_path, include
     2. Add a URL to urlpatterns:  re_path(r'^blog/', include('blog.urls'))
 """
+
 from django.contrib import admin
 from django.http import HttpResponseRedirect
 from django.urls import include, re_path

backend/docker/requirements.in

@@ -1,7 +1,8 @@
 # main dependencies for the backend application
+black>=24.3.0
 celery
-cryptography>=41.0.6
-Django>=4.2.7
+cryptography>=42.0.4
+Django>=4.2.11
 django-allauth
 django-configurations
 django-extensions
@@ -12,18 +13,20 @@ django-formtools
 django-rest-swagger
 django-webpack-loader
 djangorestframework-jwt
+idna>=3.7
 jinja2>=3.1.3
 openpyxl
 pandas
-Pillow
+Pillow>=10.3.0
 pyodbc
 redis
 requests
 scikit-learn
 sentence_transformers==2.2.1 # starting at 2.3.0 sentence_transformers requires upgrading torch to 1.11.0 - which appears to be a significant change
 statsmodels
 sqlalchemy
-transformers>=4.36.0
+sqlparse>=0.5.0
+transformers>=4.38.0
 xlrd
 
 # dependencies for code formatters, dependency management, testing, etc.

backend/docker/requirements.txt

@@ -16,7 +16,7 @@ backports-zoneinfo==0.2.1
     # via django
 billiard==3.6.4.0
     # via celery
-black==21.11b1
+black==24.4.0
     # via -r requirements.in
 build==1.0.3
     # via pip-tools
@@ -55,7 +55,7 @@ coreschema==0.0.4
     # via coreapi
 coverage==6.1.2
     # via -r requirements.in
-cryptography==42.0.2
+cryptography==42.0.5
     # via
     #   -r requirements.in
     #   pyjwt
@@ -65,7 +65,7 @@ distlib==0.3.3
     # via virtualenv
 dj-rest-auth==3.0.0
     # via -r requirements.in
-django==4.2.9
+django==4.2.11
     # via
     #   -r requirements.in
     #   dj-rest-auth
@@ -122,8 +122,10 @@ huggingface-hub==0.20.3
     #   transformers
 identify==2.3.6
     # via pre-commit
-idna==3.3
-    # via requests
+idna==3.7
+    # via
+    #   -r requirements.in
+    #   requests
 importlib-metadata==7.0.1
     # via build
 iniconfig==1.1.1
@@ -168,8 +170,9 @@ openapi-codec==1.3.2
     # via django-rest-swagger
 openpyxl==3.0.9
     # via -r requirements.in
-packaging==21.2
+packaging==24.0
     # via
+    #   black
     #   build
     #   huggingface-hub
     #   pytest
@@ -182,7 +185,7 @@ pathspec==0.9.0
     # via black
 patsy==0.5.2
     # via statsmodels
-pillow==10.2.0
+pillow==10.3.0
     # via
     #   -r requirements.in
     #   torchvision
@@ -210,11 +213,8 @@ pyjwt[crypto]==1.7.1
     # via
     #   django-allauth
     #   djangorestframework-jwt
-    #   pyjwt
 pyodbc==4.0.35
     # via -r requirements.in
-pyparsing==2.4.7
-    # via packaging
 pyproject-hooks==1.0.0
     # via build
 pytest==7.3.1
@@ -239,7 +239,6 @@ redis==5.0.1
     # via -r requirements.in
 regex==2021.11.10
     # via
-    #   black
     #   nltk
     #   transformers
 requests==2.28.2
@@ -277,8 +276,10 @@ six==1.16.0
     #   virtualenv
 sqlalchemy==1.4.28
     # via -r requirements.in
-sqlparse==0.4.2
-    # via django
+sqlparse==0.5.0
+    # via
+    #   -r requirements.in
+    #   django
 statsmodels==0.13.1
     # via -r requirements.in
 threadpoolctl==3.0.0
@@ -306,11 +307,11 @@ tqdm==4.62.3
     #   nltk
     #   sentence-transformers
     #   transformers
-transformers==4.37.2
+transformers==4.39.3
     # via
     #   -r requirements.in
     #   sentence-transformers
-typing-extensions==4.0.0
+typing-extensions==4.11.0
     # via
     #   black
     #   huggingface-hub