Merge pull request #134 from RTIInternational/master_update_dependencies

update dependencies to match npsas_main

README.md

@@ -64,6 +64,14 @@ If your database is blank, you will need to run migrations to initialize all the
 docker-compose run --rm smart_backend ./migrate.sh
 ```
 
+#### Dependency management in Python
+
+We use [pip-tools](https://github.com/jazzband/pip-tools) to manage Python dependencies. To change the dependencies:
+
+1. Edit [requirements.in](./backend/docker/requirements.in) to add, remove, or edit a dependency. You only need to put primary dependencies here, that is, the ones explicitly needed by our source code. pip-tools will take care of adding their dependencies.
+1. Run `docker-compose run --rm smart_backend pip-compile docker/requirements.in` to generate a new [requirements.txt](./backend/docker/requirements.txt). Note that pip-tools uses the existing `requirements.txt` file when building a new one, so that it can maintain existing versions. To upgrade a package to the newest version compatible with the other libraries, just remove it from the existing `requirements.txt` before running pip-compile.
+1. Run `docker-compose build smart_backend` to install the updated requirements into the Docker image.
+
 ### Custom Environment Variables
 
 The various services will be available on your machine at their standard ports, but you can override the port numbers if they conflict with other running services. For example, you don't want to run SMART's instance of Postgres on port 5432 if you already have your own local instance of Postgres running on port 5432. To override a port, create a file named `.env` in the `envs/dev` directory that looks something like this:

backend/django/smart/settings.py

@@ -16,6 +16,8 @@
 # Build paths inside the project like this: os.path.join(BASE_DIR, ...)
 BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 
+DEFAULT_AUTO_FIELD = "django.db.models.AutoField"
+
 
 # Quick-start development settings - unsuitable for production
 # See https://docs.djangoproject.com/en/1.11/howto/deployment/checklist/

backend/Dockerfile → backend/docker/Dockerfile

@@ -2,7 +2,7 @@ FROM python:3.8
 ENV PYTHONUNBUFFERED 1
 RUN apt-get update && apt-get install -y netcat-traditional
 WORKDIR /code
-ADD ./requirements.txt /code/requirements.txt
+ADD ./docker/requirements.txt /code/requirements.txt
 RUN pip install --upgrade pip
 RUN pip install -r requirements.txt
 RUN mkdir -p /data/data_files /data/tf_idf /data/model_pickles /data/code_books

backend/Dockerfile.prod → backend/docker/Dockerfile.prod

@@ -2,7 +2,7 @@ FROM python:3.5.7
 ENV PYTHONUNBUFFERED 1
 RUN apt-get update && apt-get install -y netcat-traditional mysql-client
 WORKDIR /code
-ADD ./requirements.txt /code/requirements.txt
+ADD ./docker/requirements.txt /code/requirements.txt
 RUN pip install  -r requirements.txt
 RUN pip install gunicorn
 ADD ./smart/ /code/

backend/docker/requirements.in

@@ -0,0 +1,30 @@
+# main dependencies for the backend application
+celery
+Django
+django-allauth
+django-configurations
+django-extensions
+django-rest-auth
+django-celery-results
+django-postgres-stats
+django-formtools
+django-rest-swagger
+django-webpack-loader
+djangorestframework-jwt
+openpyxl
+pandas
+redis==3.5.3
+scikit-learn==0.24.1
+statsmodels
+string-grouper
+xlrd
+
+# dependencies for code formatters, dependency management, testing, etc.
+black
+coverage
+docformatter
+flake8
+isort
+pip-tools
+pre-commit
+pytest-django

backend/docker/requirements.txt

@@ -0,0 +1,276 @@
+#
+# This file is autogenerated by pip-compile with python 3.8
+# To update, run:
+#
+#    pip-compile docker/requirements.in
+#
+amqp==5.0.6
+    # via kombu
+asgiref==3.4.1
+    # via django
+attrs==21.2.0
+    # via pytest
+backports.entry-points-selectable==1.1.1
+    # via virtualenv
+billiard==3.6.4.0
+    # via celery
+black==21.11b1
+    # via -r docker/requirements.in
+celery==5.2.2
+    # via
+    #   -r docker/requirements.in
+    #   django-celery-results
+certifi==2021.10.8
+    # via requests
+cffi==1.15.0
+    # via cryptography
+cfgv==3.3.1
+    # via pre-commit
+charset-normalizer==2.0.7
+    # via requests
+click==8.0.3
+    # via
+    #   black
+    #   celery
+    #   click-didyoumean
+    #   click-plugins
+    #   click-repl
+    #   pip-tools
+click-didyoumean==0.3.0
+    # via celery
+click-plugins==1.1.1
+    # via celery
+click-repl==0.2.0
+    # via celery
+coreapi==2.3.3
+    # via
+    #   django-rest-swagger
+    #   openapi-codec
+coreschema==0.0.4
+    # via coreapi
+coverage==6.1.2
+    # via -r docker/requirements.in
+cryptography==35.0.0
+    # via pyjwt
+cython==0.29.24
+    # via
+    #   sparse-dot-topn-for-blocks
+    #   topn
+defusedxml==0.7.1
+    # via python3-openid
+distlib==0.3.3
+    # via virtualenv
+django==3.2.9
+    # via
+    #   -r docker/requirements.in
+    #   django-allauth
+    #   django-configurations
+    #   django-extensions
+    #   django-formtools
+    #   django-postgres-stats
+    #   django-rest-auth
+    #   djangorestframework
+django-allauth==0.46.0
+    # via -r docker/requirements.in
+django-celery-results==2.2.0
+    # via -r docker/requirements.in
+django-configurations==2.3.1
+    # via -r docker/requirements.in
+django-extensions==3.1.5
+    # via -r docker/requirements.in
+django-formtools==2.3
+    # via -r docker/requirements.in
+django-postgres-stats==1.0.0
+    # via -r docker/requirements.in
+django-rest-auth==0.9.5
+    # via -r docker/requirements.in
+django-rest-swagger==2.2.0
+    # via -r docker/requirements.in
+django-webpack-loader==1.4.1
+    # via -r docker/requirements.in
+djangorestframework==3.12.4
+    # via
+    #   django-rest-auth
+    #   django-rest-swagger
+djangorestframework-jwt==1.11.0
+    # via -r docker/requirements.in
+docformatter==1.4
+    # via -r docker/requirements.in
+et-xmlfile==1.1.0
+    # via openpyxl
+filelock==3.4.0
+    # via virtualenv
+flake8==4.0.1
+    # via -r docker/requirements.in
+identify==2.3.6
+    # via pre-commit
+idna==3.3
+    # via requests
+iniconfig==1.1.1
+    # via pytest
+isort==5.10.1
+    # via -r docker/requirements.in
+itypes==1.2.0
+    # via coreapi
+jinja2==3.0.3
+    # via coreschema
+joblib==1.1.0
+    # via scikit-learn
+kombu==5.2.2
+    # via celery
+markupsafe==2.0.1
+    # via jinja2
+mccabe==0.6.1
+    # via flake8
+mypy-extensions==0.4.3
+    # via black
+nodeenv==1.6.0
+    # via pre-commit
+numpy==1.21.4
+    # via
+    #   pandas
+    #   patsy
+    #   scikit-learn
+    #   scipy
+    #   sparse-dot-topn-for-blocks
+    #   statsmodels
+    #   string-grouper
+    #   topn
+oauthlib==3.1.1
+    # via requests-oauthlib
+openapi-codec==1.3.2
+    # via django-rest-swagger
+openpyxl==3.0.9
+    # via -r docker/requirements.in
+packaging==21.2
+    # via pytest
+pandas==1.3.4
+    # via
+    #   -r docker/requirements.in
+    #   statsmodels
+    #   string-grouper
+pathspec==0.9.0
+    # via black
+patsy==0.5.2
+    # via statsmodels
+pep517==0.12.0
+    # via pip-tools
+pip-tools==6.4.0
+    # via -r docker/requirements.in
+platformdirs==2.4.0
+    # via
+    #   black
+    #   virtualenv
+pluggy==1.0.0
+    # via pytest
+pre-commit==2.15.0
+    # via -r docker/requirements.in
+prompt-toolkit==3.0.22
+    # via click-repl
+psycopg2==2.9.2
+    # via django-postgres-stats
+py==1.11.0
+    # via pytest
+pycodestyle==2.8.0
+    # via flake8
+pycparser==2.21
+    # via cffi
+pyflakes==2.4.0
+    # via flake8
+pyjwt[crypto]==1.7.1
+    # via
+    #   django-allauth
+    #   djangorestframework-jwt
+pyparsing==2.4.7
+    # via packaging
+pytest==6.2.5
+    # via pytest-django
+pytest-django==4.4.0
+    # via -r docker/requirements.in
+python-dateutil==2.8.2
+    # via pandas
+python3-openid==3.2.0
+    # via django-allauth
+pytz==2021.3
+    # via
+    #   celery
+    #   django
+    #   pandas
+pyyaml==6.0
+    # via pre-commit
+redis==3.5.3
+    # via -r docker/requirements.in
+regex==2021.11.10
+    # via black
+requests==2.26.0
+    # via
+    #   coreapi
+    #   django-allauth
+    #   requests-oauthlib
+requests-oauthlib==1.3.0
+    # via django-allauth
+scikit-learn==0.24.1
+    # via
+    #   -r docker/requirements.in
+    #   string-grouper
+scipy==1.7.2
+    # via
+    #   scikit-learn
+    #   sparse-dot-topn-for-blocks
+    #   statsmodels
+    #   string-grouper
+    #   topn
+simplejson==3.17.6
+    # via django-rest-swagger
+six==1.16.0
+    # via
+    #   click-repl
+    #   django-rest-auth
+    #   patsy
+    #   python-dateutil
+    #   virtualenv
+sparse-dot-topn-for-blocks==0.3.1.post3
+    # via string-grouper
+sqlparse==0.4.2
+    # via django
+statsmodels==0.13.1
+    # via -r docker/requirements.in
+string-grouper==0.6.1
+    # via -r docker/requirements.in
+threadpoolctl==3.0.0
+    # via scikit-learn
+toml==0.10.2
+    # via
+    #   pre-commit
+    #   pytest
+tomli==1.2.2
+    # via
+    #   black
+    #   pep517
+topn==0.0.7
+    # via string-grouper
+typing-extensions==4.0.0
+    # via black
+untokenize==0.1.1
+    # via docformatter
+uritemplate==4.1.1
+    # via coreapi
+urllib3==1.26.7
+    # via requests
+vine==5.0.0
+    # via
+    #   amqp
+    #   celery
+    #   kombu
+virtualenv==20.10.0
+    # via pre-commit
+wcwidth==0.2.5
+    # via prompt-toolkit
+wheel==0.37.0
+    # via pip-tools
+xlrd==2.0.1
+    # via -r docker/requirements.in
+
+# The following packages are considered to be unsafe in a requirements file:
+# pip
+# setuptools