From 2c5240f4cbc2eb5bedc30742e65d3702d159bf45 Mon Sep 17 00:00:00 2001 From: Bruno Pereira Date: Thu, 25 Nov 2021 17:52:02 -0500 Subject: [PATCH 01/33] Fix captcha validation --- api/resources/api_manager.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/api/resources/api_manager.py b/api/resources/api_manager.py index 3073389b..de9b66e0 100644 --- a/api/resources/api_manager.py +++ b/api/resources/api_manager.py @@ -192,7 +192,9 @@ def post(self): if request.method == "POST": json = request.get_json() value = json["response"] - key = os.environ.get("CAPTCHA_KEY_FILE") + with open(CAPTCHA_KEY_FILE, "rb") as f: + for line in f: + key = line if key: ret = requests.post( "https://www.google.com/recaptcha/api/siteverify", From ced5482234116d9c9974110cdb70f35161affa6d Mon Sep 17 00:00:00 2001 From: asherpasha Date: Fri, 26 Nov 2021 20:16:30 -0500 Subject: [PATCH 02/33] Documentation initial commit --- .gitignore | 2 ++ docs/Makefile | 20 ++++++++++++++++ docs/make.bat | 35 +++++++++++++++++++++++++++ docs/source/conf.py | 55 +++++++++++++++++++++++++++++++++++++++++++ docs/source/index.rst | 30 +++++++++++++++++++++++ requirements-docs.txt | 23 ++++++++++++++++++ 6 files changed, 165 insertions(+) create mode 100644 docs/Makefile create mode 100644 docs/make.bat create mode 100644 docs/source/conf.py create mode 100644 docs/source/index.rst create mode 100644 requirements-docs.txt diff --git a/.gitignore b/.gitignore index c5e6bab0..ca846006 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ __pycache__/ build/ develop-eggs/ dist/ +docs/build/ downloads/ eggs/ .eggs/ @@ -106,6 +107,7 @@ celerybeat.pid .venv env/ venv/ +venv-docs/ ENV/ env.bak/ venv.bak/ diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 00000000..d0c3cbf1 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = source +BUILDDIR = build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 00000000..6fcf05b4 --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=source +set BUILDDIR=build + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.https://www.sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/docs/source/conf.py b/docs/source/conf.py new file mode 100644 index 00000000..76e4f39a --- /dev/null +++ b/docs/source/conf.py @@ -0,0 +1,55 @@ +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +# import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) + + +# -- Project information ----------------------------------------------------- + +project = 'BAR API' +copyright = '2021, BAR Developers' +author = 'BAR Developers' + +# The full version, including alpha/beta/rc tags +release = '0.0.1' + + +# -- General configuration --------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = [] + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'alabaster' + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] \ No newline at end of file diff --git a/docs/source/index.rst b/docs/source/index.rst new file mode 100644 index 00000000..8a6a4b05 --- /dev/null +++ b/docs/source/index.rst @@ -0,0 +1,30 @@ +.. BAR API documentation master file, created by + sphinx-quickstart on Fri Nov 26 19:30:59 2021. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to BAR API's documentation! +=================================== + +This is the documentation for the BAR API. + +Run on your own computer with Docker +------------------------------------ +1. Install `Docker`_ +2. Install `Docker Compose`_ +3. Install `Git`_ +4. Clone this repository and change directory to ``BAR_API`` +5. Build docker images + + ``docker-compose build`` + +6. Run docker containers (-d is detached) + + ``docker-compose up -d`` + +7. Load ``http://localhost:5000/`` in a web browser. Enjoy :) + +.. _Docker: https://docs.docker.com/get-docker/ +.. _Docker Compose: https://docs.docker.com/compose/install/ +.. _Git: https://git-scm.com/downloads + diff --git a/requirements-docs.txt b/requirements-docs.txt new file mode 100644 index 00000000..e983264b --- /dev/null +++ b/requirements-docs.txt @@ -0,0 +1,23 @@ +alabaster==0.7.12 +Babel==2.9.1 +certifi==2021.10.8 +charset-normalizer==2.0.8 +docutils==0.17.1 +idna==3.3 +imagesize==1.3.0 +Jinja2==3.0.3 +MarkupSafe==2.0.1 +packaging==21.3 +Pygments==2.10.0 +pyparsing==3.0.6 +pytz==2021.3 +requests==2.26.0 +snowballstemmer==2.2.0 +Sphinx==4.3.0 +sphinxcontrib-applehelp==1.0.2 +sphinxcontrib-devhelp==1.0.2 +sphinxcontrib-htmlhelp==2.0.0 +sphinxcontrib-jsmath==1.0.1 +sphinxcontrib-qthelp==1.0.3 +sphinxcontrib-serializinghtml==1.1.5 +urllib3==1.26.7 From 07fedda252f91d732a55ae02639b05c41e776670 Mon Sep 17 00:00:00 2001 From: asherpasha Date: Fri, 26 Nov 2021 20:41:44 -0500 Subject: [PATCH 03/33] Updated documentation. --- README.md | 73 ++----------------------------------------- docs/source/index.rst | 55 ++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 71 deletions(-) diff --git a/README.md b/README.md index 8c2d986d..2278f4ff 100644 --- a/README.md +++ b/README.md @@ -3,75 +3,6 @@ **master**: [![Build Status](https://github.com/BioAnalyticResource/BAR_API/workflows/BAR-API/badge.svg?branch=master)](https://github.com/BioAnalyticResource/BAR_API/actions?query=branch%3Amaster) [![codecov](https://codecov.io/gh/BioAnalyticResource/BAR_API/branch/master/graph/badge.svg?token=QSYUWTRYEV)](https://codecov.io/gh/BioAnalyticResource/BAR_API) **dev**: [![Build Status](https://github.com/BioAnalyticResource/BAR_API/workflows/BAR-API/badge.svg?branch=dev)](https://github.com/BioAnalyticResource/BAR_API/actions?query=branch%3Adev) [![codecov](https://codecov.io/gh/BioAnalyticResource/BAR_API/branch/dev/graph/badge.svg?token=QSYUWTRYEV)](https://codecov.io/gh/BioAnalyticResource/BAR_API) [![Website Status](https://img.shields.io/website?url=http%3A%2F%2Fbar.utoronto.ca%2Fapi%2F)](http://bar.utoronto.ca/api/) ![GitHub repo size](https://img.shields.io/github/repo-size/BioAnalyticResource/BAR_API) [![LGTM Alerts](https://img.shields.io/lgtm/alerts/github/BioAnalyticResource/BAR_API)](https://lgtm.com/projects/g/BioAnalyticResource/BAR_API/?mode=list) [![LGTM -Grade](https://img.shields.io/lgtm/grade/python/github/BioAnalyticResource/BAR_API)](https://lgtm.com/projects/g/BioAnalyticResource/BAR_API/latest/files/?sort=name&dir=ASC&mode=heatmap) [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) +Grade](https://img.shields.io/lgtm/grade/python/github/BioAnalyticResource/BAR_API)](https://lgtm.com/projects/g/BioAnalyticResource/BAR_API/latest/files/?sort=name&dir=ASC&mode=heatmap) [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) [![Documentation Status](https://readthedocs.org/projects/bar-api/badge/?version=latest)](https://bar-api.readthedocs.io/en/latest/?badge=latest) -This is the official repository for the Bio-Analytic Resource API. - -## Status - -Apart from Travis CI and testing on the live BAR, we frequently test the API on systems that are not BAR for cross-platform compatibility. The most recent test is on the following systems: - -* OpenBSD 6.7-CURRENT (Maria DB 10.4.12v1, Python 3.8.3 and Redis 6.0.5) -* FreeBSD 12.1-RELEASE-p6 (MySQL 8.0.20, Python 3.8.3 and Redis 5.0.9) - -## Run on your own computer with Docker - -1. Install [Docker](https://docs.docker.com/get-docker/) -2. Install [Docker Compose](https://docs.docker.com/compose/install/) -3. Install [Git](https://git-scm.com/downloads) -4. Clone this repository and change directory to ```BAR_API``` -5. Build docker images -``` -docker-compose build -``` -6. Run docker containers (-d is detached) -``` -docker-compose up -d -``` -7. Load ```http://localhost:5000/``` in a web browser. Enjoy :) - -## Run on your own computer without Docker - -1. Install [MySQL](https://www.mysql.com/products/community/) or [Maria DB](https://mariadb.com/downloads/). -2. Install [Redis](https://redis.io/download). -3. Install [Python](https://www.python.org/downloads/) or [Pypy](https://www.pypy.org/download.html). Note: Python 2 is not supported. -4. Install [Git](https://git-scm.com/downloads) -5. (Optional) On Debian based systems, you may also need to install ```libmysqlclient-dev``` and ```python3-dev```. On FreeBSD, you may need to install ```py38-sqlite3```. We will update this step as we come across more OS dependencies. -6. Clone this repository and change directory to ```BAR_API``` -7. Set up a virtual environment -``` -python3 -m venv venv -``` -8. Activate the virtual environment. Bash/Zsh: -``` -source venv/bin/activate -``` -csh/tcsh: -``` -source venv/bin/activate.csh -``` -9. Install requirements -``` -pip install --upgrade pip setuptools wheel -pip install -r requirements.txt -``` -10. Copy ```config/BAR_API.cgi``` to your preferred directory, for example: -``` -cp config/BAR_API.cgi ~/.config/ -``` -Add, update, and modify passwords and environment variables as needed. - -11. Copy ```./config/init.sh``` to BAR_API directory: -``` -cp config/init.sh . -``` -Change passwords in ```./init.sh``` and run this script to load the databases: -``` -./init.sh -``` -Then delete ```./init.sh``` - -12. Edit ```./api/__init__.py``` and update the location of your BAR_API.cfg file if you have changed it. -13. Run ```pytest```. Tests should pass if the system is set up correctly. -14. Run ```python app.py``` to start. -15. Load ```http://localhost:5000/``` in a web browser. Enjoy :) +This is the official repository for the Bio-Analytic Resource API. The API documentation can be found [here](https://bar-api.readthedocs.io/en/latest/). diff --git a/docs/source/index.rst b/docs/source/index.rst index 8a6a4b05..2d9a2225 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -28,3 +28,58 @@ Run on your own computer with Docker .. _Docker Compose: https://docs.docker.com/compose/install/ .. _Git: https://git-scm.com/downloads +Run on your own computer without Docker +--------------------------------------- + +1. Install `MySQL`_ or `Maria DB`_ +2. Install `Redis`_ +3. Install `Python`_ or `Pypy`_. Note: Python 2 is not supported. +4. Install [Git](https://git-scm.com/downloads) +5. (Optional) On Debian based systems, you may also need to install ``libmysqlclient-dev`` and ``python3-dev``. On FreeBSD, you may need to install ``py38-sqlite3``. We will update this step as we come across more OS dependencies. +6. Clone this repository and change directory to ``BAR_API`` +7. Set up a virtual environment + + ``python3 -m venv venv`` + +8. Activate the virtual environment. Bash/Zsh: + + ``source venv/bin/activate`` + +csh/tcsh: + + ``source venv/bin/activate.csh`` + +9. Install requirements + + ``pip install --upgrade pip setuptools wheel`` + ``pip install -r requirements.txt`` + +10. Copy ``config/BAR_API.cgi`` to your preferred directory, for example: + + ``cp config/BAR_API.cgi ~/.config/`` + +Add, update, and modify passwords and environment variables as needed. + +11. Copy ``./config/init.sh`` to BAR_API directory: + + ``cp config/init.sh .`` + +Change passwords in ``./init.sh`` and run this script to load the databases: + + ``./init.sh`` + +Then delete ``./init.sh`` + +12. Edit ``./api/__init__.py`` and update the location of your BAR_API.cfg file if you have changed it. + +13. Run ``pytest``. Tests should pass if the system is set up correctly. + +14. Run ``python app.py`` to start. + +15. Load ``http://localhost:5000/`` in a web browser. Enjoy :) + +.. _MySQL: https://www.mysql.com/products/community/ +.. _Maria DB: https://mariadb.com/downloads/ +.. _Redis: https://redis.io/download +.. _Python: https://www.python.org/downloads/ +.. _Pypy: https://www.pypy.org/download.html \ No newline at end of file From 0504f57099a56f06c9a150874dff0b64f697f4d2 Mon Sep 17 00:00:00 2001 From: asherpasha Date: Sun, 28 Nov 2021 14:49:59 -0500 Subject: [PATCH 04/33] Updated documentation. --- .../requirements.txt | 7 +- docs/source/conf.py | 3 +- docs/source/developer_guide.rst | 4 + docs/source/index.rst | 79 +-------------- docs/source/installation_guide.rst | 95 +++++++++++++++++++ docs/source/user_guide.rst | 4 + 6 files changed, 116 insertions(+), 76 deletions(-) rename requirements-docs.txt => docs/requirements.txt (79%) create mode 100644 docs/source/developer_guide.rst create mode 100644 docs/source/installation_guide.rst create mode 100644 docs/source/user_guide.rst diff --git a/requirements-docs.txt b/docs/requirements.txt similarity index 79% rename from requirements-docs.txt rename to docs/requirements.txt index e983264b..d661ed5f 100644 --- a/requirements-docs.txt +++ b/docs/requirements.txt @@ -1,19 +1,24 @@ alabaster==0.7.12 Babel==2.9.1 +beautifulsoup4==4.10.0 certifi==2021.10.8 charset-normalizer==2.0.8 docutils==0.17.1 +furo==2021.11.23 idna==3.3 imagesize==1.3.0 Jinja2==3.0.3 MarkupSafe==2.0.1 packaging==21.3 +pkg_resources==0.0.0 Pygments==2.10.0 pyparsing==3.0.6 pytz==2021.3 requests==2.26.0 snowballstemmer==2.2.0 -Sphinx==4.3.0 +soupsieve==2.3.1 +Sphinx==4.3.1 +sphinx-copybutton==0.4.0 sphinxcontrib-applehelp==1.0.2 sphinxcontrib-devhelp==1.0.2 sphinxcontrib-htmlhelp==2.0.0 diff --git a/docs/source/conf.py b/docs/source/conf.py index 76e4f39a..721f59ad 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -31,6 +31,7 @@ # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ + 'sphinx_copybutton' ] # Add any paths that contain templates here, relative to this directory. @@ -47,7 +48,7 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = 'alabaster' +html_theme = 'furo' # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, diff --git a/docs/source/developer_guide.rst b/docs/source/developer_guide.rst new file mode 100644 index 00000000..67f12e43 --- /dev/null +++ b/docs/source/developer_guide.rst @@ -0,0 +1,4 @@ +Developer Guide +=============== + +Coming soon ... \ No newline at end of file diff --git a/docs/source/index.rst b/docs/source/index.rst index 2d9a2225..7d8cc810 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -8,78 +8,9 @@ Welcome to BAR API's documentation! This is the documentation for the BAR API. -Run on your own computer with Docker ------------------------------------- -1. Install `Docker`_ -2. Install `Docker Compose`_ -3. Install `Git`_ -4. Clone this repository and change directory to ``BAR_API`` -5. Build docker images +.. toctree:: + :maxdepth: 2 - ``docker-compose build`` - -6. Run docker containers (-d is detached) - - ``docker-compose up -d`` - -7. Load ``http://localhost:5000/`` in a web browser. Enjoy :) - -.. _Docker: https://docs.docker.com/get-docker/ -.. _Docker Compose: https://docs.docker.com/compose/install/ -.. _Git: https://git-scm.com/downloads - -Run on your own computer without Docker ---------------------------------------- - -1. Install `MySQL`_ or `Maria DB`_ -2. Install `Redis`_ -3. Install `Python`_ or `Pypy`_. Note: Python 2 is not supported. -4. Install [Git](https://git-scm.com/downloads) -5. (Optional) On Debian based systems, you may also need to install ``libmysqlclient-dev`` and ``python3-dev``. On FreeBSD, you may need to install ``py38-sqlite3``. We will update this step as we come across more OS dependencies. -6. Clone this repository and change directory to ``BAR_API`` -7. Set up a virtual environment - - ``python3 -m venv venv`` - -8. Activate the virtual environment. Bash/Zsh: - - ``source venv/bin/activate`` - -csh/tcsh: - - ``source venv/bin/activate.csh`` - -9. Install requirements - - ``pip install --upgrade pip setuptools wheel`` - ``pip install -r requirements.txt`` - -10. Copy ``config/BAR_API.cgi`` to your preferred directory, for example: - - ``cp config/BAR_API.cgi ~/.config/`` - -Add, update, and modify passwords and environment variables as needed. - -11. Copy ``./config/init.sh`` to BAR_API directory: - - ``cp config/init.sh .`` - -Change passwords in ``./init.sh`` and run this script to load the databases: - - ``./init.sh`` - -Then delete ``./init.sh`` - -12. Edit ``./api/__init__.py`` and update the location of your BAR_API.cfg file if you have changed it. - -13. Run ``pytest``. Tests should pass if the system is set up correctly. - -14. Run ``python app.py`` to start. - -15. Load ``http://localhost:5000/`` in a web browser. Enjoy :) - -.. _MySQL: https://www.mysql.com/products/community/ -.. _Maria DB: https://mariadb.com/downloads/ -.. _Redis: https://redis.io/download -.. _Python: https://www.python.org/downloads/ -.. _Pypy: https://www.pypy.org/download.html \ No newline at end of file + installation_guide + user_guide + developer_guide \ No newline at end of file diff --git a/docs/source/installation_guide.rst b/docs/source/installation_guide.rst new file mode 100644 index 00000000..e3b78a1e --- /dev/null +++ b/docs/source/installation_guide.rst @@ -0,0 +1,95 @@ +Installation Guide +================== + +Run on your own computer with Docker +------------------------------------ +1. Install `Docker`_ +2. Install `Docker Compose`_ +3. Install `Git`_ +4. Clone this repository and change directory to ``BAR_API`` +5. Build docker images + +.. code-block:: bash + + docker-compose build + +6. Run docker containers (-d is detached) + +.. code-block:: bash + + docker-compose up -d + +7. Load ``http://localhost:5000/`` in a web browser. Enjoy :) + +Run on your own computer without Docker +--------------------------------------- + +1. Install `MySQL`_ or `Maria DB`_ +2. Install `Redis`_ +3. Install `Python`_ or `Pypy`_. Note: Python 2 is not supported. +4. Install `Git`_ +5. (Optional) On Debian based systems, you may also need to install ``libmysqlclient-dev`` and ``python3-dev``. On FreeBSD, you may need to install ``py38-sqlite3``. We will update this step as we come across more OS dependencies. +6. Clone this repository and change directory to ``BAR_API`` +7. Set up a virtual environment + +.. code-block:: bash + + python3 -m venv venv + +8. Activate the virtual environment. Bash/Zsh: + +.. code-block:: bash + + source venv/bin/activate + +csh/tcsh: + +.. code-block:: bash + + source venv/bin/activate.csh + +9. Install requirements + +.. code-block:: bash + + pip install --upgrade pip setuptools wheel + pip install -r requirements.txt + +10. Copy ``config/BAR_API.cgi`` to your preferred directory, for example: + +.. code-block:: bash + + cp config/BAR_API.cgi ~/.config/ + +Add, update, and modify passwords and environment variables as needed. + +11. Copy ``./config/init.sh`` to BAR_API directory: + +.. code-block:: bash + + cp config/init.sh . + +Change passwords in ``./init.sh`` and run this script to load the databases: + +.. code-block:: bash + + ./init.sh + +Then delete ``./init.sh``. + +12. Edit ``./api/__init__.py`` and update the location of your BAR_API.cfg file if you have changed it. + +13. Run ``pytest``. Tests should pass if the system is set up correctly. + +14. Run ``python app.py`` to start. + +15. Load ``http://localhost:5000/`` in a web browser. Enjoy :) + +.. _Docker: https://docs.docker.com/get-docker/ +.. _Docker Compose: https://docs.docker.com/compose/install/ +.. _Git: https://git-scm.com/downloads +.. _MySQL: https://www.mysql.com/products/community/ +.. _Maria DB: https://mariadb.com/downloads/ +.. _Redis: https://redis.io/download +.. _Python: https://www.python.org/downloads/ +.. _Pypy: https://www.pypy.org/download.html diff --git a/docs/source/user_guide.rst b/docs/source/user_guide.rst new file mode 100644 index 00000000..c4570fd4 --- /dev/null +++ b/docs/source/user_guide.rst @@ -0,0 +1,4 @@ +User Guide +========== + +Coming soon ... \ No newline at end of file From 83ba58fb466cf8afdc55555116ed20743a987ffa Mon Sep 17 00:00:00 2001 From: asherpasha Date: Sun, 28 Nov 2021 15:08:22 -0500 Subject: [PATCH 05/33] Minor bug fix --- docs/requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index d661ed5f..c74e6c05 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -10,7 +10,6 @@ imagesize==1.3.0 Jinja2==3.0.3 MarkupSafe==2.0.1 packaging==21.3 -pkg_resources==0.0.0 Pygments==2.10.0 pyparsing==3.0.6 pytz==2021.3 From e9fa9b18549aa98e3925996b9b41328cfa4c1549 Mon Sep 17 00:00:00 2001 From: Vin Date: Wed, 1 Dec 2021 13:24:03 -0500 Subject: [PATCH 06/33] added unit tests, standardized response JSON for interactions and localization endpoints --- api/resources/gene_localizations.py | 4 +- api/resources/interactions.py | 4 +- tests/resources/test_interactions.py | 65 +++++++++++++ tests/resources/test_localizations.py | 133 ++++++++++++++++++++++++++ 4 files changed, 202 insertions(+), 4 deletions(-) create mode 100644 tests/resources/test_interactions.py create mode 100644 tests/resources/test_localizations.py diff --git a/api/resources/gene_localizations.py b/api/resources/gene_localizations.py index 28464764..f638d3f5 100644 --- a/api/resources/gene_localizations.py +++ b/api/resources/gene_localizations.py @@ -46,8 +46,8 @@ def get(self, species="", query_gene=""): else: print(rows) return { - "status": "success", - "result": { + "wasSuccessful": True, + "data": { "gene": rows[0].gene_id, "predicted_location": rows[0].pred_mPLoc, } diff --git a/api/resources/interactions.py b/api/resources/interactions.py index 622fcd2c..21dfea13 100644 --- a/api/resources/interactions.py +++ b/api/resources/interactions.py @@ -51,8 +51,8 @@ def get(self, species="", query_gene=""): for i in rows ] return { - "status": "success", - "result": res + "wasSuccessful": True, + "data": res } except OperationalError: return BARUtils.error_exit("An internal error has occurred"), 500 diff --git a/tests/resources/test_interactions.py b/tests/resources/test_interactions.py new file mode 100644 index 00000000..a8c0fb99 --- /dev/null +++ b/tests/resources/test_interactions.py @@ -0,0 +1,65 @@ +from api import app +from unittest import TestCase + + +class TestIntegrations(TestCase): + + maxDiff = None + + def setUp(self): + self.app_client = app.test_client() + + def test_get_itrns(self): + """ + This function test retrieving protein interactions for various species' genes. + """ + + # Valid request rice + response = self.app_client.get("/interactions/rice/LOC_Os01g52560") + expected = { + "wasSuccessful": True, + "data" : + [ + { + "protein_1": "LOC_Os01g01080", + "protein_2": "LOC_Os01g52560", + "total_hits": 1, + "Num_species": 1, + "Quality": 1, + "pcc": 0.65 + }, + { + "protein_1": "LOC_Os01g52560", + "protein_2": "LOC_Os01g73310", + "total_hits": 1, + "Num_species": 1, + "Quality": 1, + "pcc": -0.116 + } + ] + } + self.assertEqual(response.json, expected) + + # Invalid species + response = self.app_client.get("/interactions/poplar/abc") + expected = { + "wasSuccessful": False, + "error": "Invalid species or gene ID" + } + self.assertEqual(response.json, expected) + + # Invalid gene id + response = self.app_client.get("/interactions/rice/abc") + expected = { + "wasSuccessful": False, + "error": "Invalid species or gene ID" + } + self.assertEqual(response.json, expected) + + # Gene does not exist + response = self.app_client.get("/interactions/rice/LOC_Os01g52565") + expected = { + "wasSuccessful": False, + "error": "There are no data found for the given gene", + } + self.assertEqual(response.json, expected) diff --git a/tests/resources/test_localizations.py b/tests/resources/test_localizations.py new file mode 100644 index 00000000..8853baa7 --- /dev/null +++ b/tests/resources/test_localizations.py @@ -0,0 +1,133 @@ +from api import app +from unittest import TestCase +import json + + +class TestIntegrations(TestCase): + def setUp(self): + self.app_client = app.test_client() + + def test_get_loc(self): + """ + This function test retrieving subcellular localizations for various species' genes via GET. + """ + + # Valid request rice + response = self.app_client.get("/loc/rice/LOC_Os01g52560.1") + expected = { + "wasSuccessful": True, + "data" : + { + "gene": "LOC_Os01g52560.1", + "predicted_location": "Cellmembrane,Chloroplast" + } + } + self.assertEqual(response.json, expected) + + # Invalid species + response = self.app_client.get("/loc/poplar/LOC_Os01g52560.1") + expected = { + "wasSuccessful": False, + "error": "Invalid species or gene ID" + } + self.assertEqual(response.json, expected) + + # Invalid gene id + response = self.app_client.get("/loc/rice/abc") + expected = { + "wasSuccessful": False, + "error": "Invalid species or gene ID" + } + self.assertEqual(response.json, expected) + + # Gene does not exist + response = self.app_client.get("/loc/rice/LOC_Os01g52561.1") + expected = { + "wasSuccessful": False, + "error": "There are no data found for the given gene", + } + self.assertEqual(response.json, expected) + + def test_post_loc(self): + """ + This function test retrieving subcellular localizations for various species' genes via POST. + """ + + # Valid request + response = self.app_client.post( + "/loc/", + json={ + "species": "rice", + "genes": [ + "LOC_Os01g01080.1", + "LOC_Os01g52560.1" + ] + } + ) + data = json.loads(response.get_data(as_text=True)) + expected = { + "wasSuccessful": True, + "data": { + "LOC_Os01g01080.1": [ + "Endoplasmic reticulum" + ], + "LOC_Os01g52560.1": [ + "Cellmembrane,Chloroplast" + ] + } + } + self.assertEqual(data, expected) + + # Invalid species + response = self.app_client.post( + "/loc/", + json={ + "species": "poplar", + "genes": [ + "LOC_Os01g01080.1", + "LOC_Os01g52560.1" + ] + } + ) + data = json.loads(response.get_data(as_text=True)) + expected = { + "wasSuccessful": False, + "error": "Invalid species" + } + self.assertEqual(data, expected) + + # Invalid gene ID + response = self.app_client.post( + "/loc/", + json={ + "species": "rice", + "genes": [ + "abc", + "xyz" + ] + } + ) + data = json.loads(response.get_data(as_text=True)) + expected = { + "wasSuccessful": False, + "error": "Invalid gene id" + } + self.assertEqual(data, expected) + + # No data for valid gene IDs + response = self.app_client.post( + "/loc/", + json={ + "species": "rice", + "genes": [ + "LOC_Os01g01085.1", + "LOC_Os01g52565.1" + ] + } + ) + data = json.loads(response.get_data(as_text=True)) + expected = { + "wasSuccessful": False, + "error": "No data for the given species/genes" + } + self.assertEqual(data, expected) From 500ef452457cb34148204991231f6194987e25e2 Mon Sep 17 00:00:00 2001 From: Bruno Pereira Date: Wed, 1 Dec 2021 15:55:53 -0500 Subject: [PATCH 07/33] Fix API manager keys --- api/__init__.py | 4 ++-- config/BAR_API.cfg | 8 +++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/api/__init__.py b/api/__init__.py index 266f6e51..436ddf23 100644 --- a/api/__init__.py +++ b/api/__init__.py @@ -20,10 +20,10 @@ def create_app(): print("We are now loading configuration.") bar_app.config.from_pyfile(os.getcwd() + "/config/BAR_API.cfg", silent=True) if bar_app.config.get("ADMIN_ENCRYPT_KEY"): - os.environ["ADMIN_ENCRYPT_KEY"] = bar_app.config.get("ADMIN_ENCRYPT_KEY") + os.environ["ADMIN_ENCRYPT_KEY"] = bar_app.config.get("TEST_ADMIN_ENCRYPT_KEY") if bar_app.config.get("ADMIN_PASSWORD_FILE"): os.environ["ADMIN_PASSWORD_FILE"] = bar_app.config.get( - "ADMIN_PASSWORD_FILE" + "TEST_ADMIN_PASSWORD_FILE" ) elif os.environ.get("BAR"): # The BAR diff --git a/config/BAR_API.cfg b/config/BAR_API.cfg index 9a59a148..9eb8a74c 100644 --- a/config/BAR_API.cfg +++ b/config/BAR_API.cfg @@ -23,9 +23,11 @@ SQLALCHEMY_BINDS = { } ## API Manager variables -# Dummy keys for testing - use the config file in /home/bpereira/.config/ for production -ADMIN_PASSWORD_FILE = './tests/data/test_key.bin' -ADMIN_ENCRYPT_KEY = 'fZ2zIiSoV3MOlwiyhjxPWomIArj6YwuDsDNP0J68ZT8=' +ADMIN_PASSWORD_FILE = '/home/bpereira/dev/pw-script/managerkey.bin' +ADMIN_ENCRYPT_KEY = 'fabUGnTJ3UQ4qeDJbnSMrb-tDdmt9kxLkuq3GHKdGTo=' + +TEST_ADMIN_PASSWORD_FILE = './tests/data/test_key.bin' +TEST_ADMIN_ENCRYPT_KEY = 'fZ2zIiSoV3MOlwiyhjxPWomIArj6YwuDsDNP0J68ZT8=' # Google Captcha key CAPTCHA_KEY_FILE = '/home/bpereira/data/bar.summarization/key' From c92e0d619202e563369df02ba879914c06b82635 Mon Sep 17 00:00:00 2001 From: Bruno Pereira Date: Thu, 2 Dec 2021 17:24:14 -0500 Subject: [PATCH 08/33] Change DATA_FOLDER --- api/resources/summarization_gene_expression.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/resources/summarization_gene_expression.py b/api/resources/summarization_gene_expression.py index d52e2e42..2d448459 100644 --- a/api/resources/summarization_gene_expression.py +++ b/api/resources/summarization_gene_expression.py @@ -16,7 +16,7 @@ from cryptography.fernet import Fernet -DATA_FOLDER = "/home/barapps/cromwell/summarization-data" +DATA_FOLDER = "/home/bpereira/data/summarization-data" # DATA_FOLDER = '/windir/c/Users/Bruno/Documents/SummarizationCache' SUMMARIZATION_FILES_PATH = "/home/barapps/cromwell/summarization" CROMWELL_URL = "http://localhost:3020" From 72c2bc185c31deb4af20241dda0d522e37e9a0b8 Mon Sep 17 00:00:00 2001 From: Bruno Pereira Date: Fri, 3 Dec 2021 13:18:01 -0500 Subject: [PATCH 09/33] Fix requests not being removed, create table on user approval --- api/resources/api_manager.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/api/resources/api_manager.py b/api/resources/api_manager.py index de9b66e0..1bdd49c1 100644 --- a/api/resources/api_manager.py +++ b/api/resources/api_manager.py @@ -5,6 +5,7 @@ from flask_restx import Namespace, Resource from datetime import datetime from sqlalchemy.exc import SQLAlchemyError +from sqlalchemy.types import String, Float import os import uuid import requests @@ -167,17 +168,27 @@ def post(self): "date_added": datetime.now(), "status": "user", "api_key": key, - "uses_left": 25, + "uses_left": 100, } ) for row in rows ] df = pandas.DataFrame.from_records([values[0]]) + values_df = pandas.DataFrame(columns=['Gene', + 'Sample', + 'Value']) con = db.get_engine(bind="summarization") try: df.to_sql("users", con, if_exists="append", index=False) + values_df.to_sql(key, con, index_label='index', + dtype={values_df.index.name: String(42), + 'Gene': String(32), + 'Sample': String(32), + 'Value': Float}, + if_exists="append", index=True) el = table.query.filter_by(email=email).one() db.session.delete(el) + db.session.commit() except SQLAlchemyError: return BARUtils.error_exit("Internal server error"), 500 return BARUtils.success_exit(key) From e71be8b98bde1f4a0b12d2d4959a1e3ce14f501b Mon Sep 17 00:00:00 2001 From: asherpasha Date: Fri, 3 Dec 2021 17:58:46 -0500 Subject: [PATCH 10/33] Working on documentation. --- .github/workflows/bar-api.yml | 1 - docs/requirements.txt | 2 +- docs/source/developer_guide.rst | 55 ++++++++++++++++++++++++++++++++- 3 files changed, 55 insertions(+), 3 deletions(-) diff --git a/.github/workflows/bar-api.yml b/.github/workflows/bar-api.yml index 04ae5bfc..df6166a5 100644 --- a/.github/workflows/bar-api.yml +++ b/.github/workflows/bar-api.yml @@ -59,4 +59,3 @@ jobs: uses: codecov/codecov-action@v2 with: fail_ci_if_error: true - token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos diff --git a/docs/requirements.txt b/docs/requirements.txt index c74e6c05..eeb1665d 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -2,7 +2,7 @@ alabaster==0.7.12 Babel==2.9.1 beautifulsoup4==4.10.0 certifi==2021.10.8 -charset-normalizer==2.0.8 +charset-normalizer==2.0.9 docutils==0.17.1 furo==2021.11.23 idna==3.3 diff --git a/docs/source/developer_guide.rst b/docs/source/developer_guide.rst index 67f12e43..c887f61b 100644 --- a/docs/source/developer_guide.rst +++ b/docs/source/developer_guide.rst @@ -1,4 +1,57 @@ Developer Guide =============== -Coming soon ... \ No newline at end of file +Basic Requirements +------------------ +Developers need to learn Python 3 programming language (tutorial: `Python3`_ ) and GitHub. + +Core Dependencies +----------------- + +**Flask**: The BAR API uses Python Flask (tutorial: `Flask`_). We use a class based approach to organize code. + +**Flask-restx**: Flask RestX is used to create endpoints and generate swagger fontend (tutorial: `Flask-restx`_ ). + +**Flask-Caching**: This module is used to cache queries (tutorial: `Flask-Caching`_). We used Redis as a key-value store. + +**Flask-Limiter**: This module is used to rate-limit the usage of API end points. This will prevent server from overloading (tutorial: `Flask-Limiter`_ ) + +**Marshmallow**: This module is only used to validate JSON schema for POST requests (tutorial: `Marshmallow`_). + +**MarkupSafe**: This is used to validate GET request inputs (tutorial: `MarkupSafe`_). + +**Flask-SQLAlchemy**: This is used to provide database connectivity and queries (tutorial: `Flask-SQLAlchemy`_). This is based on Python SQLAlchemy (tutorial: `SQLAlchemy`_) + +**Flake8**: This is used to see if there are no style errors in code. Just run ``flake8`` command. + +**Pytest**: This module (tutorial: `Pytest`_) along with Flask testing framework (tutorial: `Flask-Testing`_) is used to unit test API endpoints. + +**Coverage**: This module show the test code coverage, that is what parts of code are tested by unit tests (tutorial: `Coverage`_) + +**Black**: This module is used to format and clean up code. Just run ``black .`` command. + +Online CI/CD Pipeline +--------------------- + +**GitHub Actions**: This runs automated testing on GitHub (tutorial: `GitHub-Actions`_). + +**LGTM**: This is for code quality testing. + +**Codecov**: This report code coverage. + +**Read the Docs**: This hosts documentation (tutorial: `readthedocs`_). + +.. _Python3: https://docs.python.org/3/tutorial/index.html +.. _Flask: https://flask.palletsprojects.com/en/2.0.x/quickstart/ +.. _Flask-Caching: https://flask-caching.readthedocs.io/en/latest/index.html +.. _Flask-Limiter: https://flask-limiter.readthedocs.io/en/master/#using-flask-pluggable-views +.. _Flask-restx: https://flask-restx.readthedocs.io/en/latest/ +.. _Marshmallow: https://marshmallow.readthedocs.io/en/stable/quickstart.html +.. _MarkupSafe: https://pypi.org/project/MarkupSafe/ +.. _Flask-SQLAlchemy: https://flask-sqlalchemy.palletsprojects.com/en/2.x/ +.. _SQLAlchemy: https://docs.sqlalchemy.org/en/14/ +.. _Pytest: https://docs.pytest.org/en/latest/getting-started.html +.. _Flask-Testing: https://flask.palletsprojects.com/en/2.0.x/testing/ +.. _Coverage: https://coverage.readthedocs.io/en/6.2/ +.. _GitHub-Actions: https://docs.github.com/en/actions/quickstart +.. _readthedocs: https://docs.readthedocs.io/en/stable/tutorial/ From fd20ec6da29f3349f8fc227a9bb1468db065e0b8 Mon Sep 17 00:00:00 2001 From: asherpasha Date: Sun, 5 Dec 2021 14:47:17 -0500 Subject: [PATCH 11/33] eFP image service initial commit. --- api/__init__.py | 2 ++ api/resources/efp_image.py | 15 +++++++++++++++ requirements.txt | 12 ++++++------ tests/resources/test_efp_image.py | 15 +++++++++++++++ 4 files changed, 38 insertions(+), 6 deletions(-) create mode 100644 api/resources/efp_image.py create mode 100644 tests/resources/test_efp_image.py diff --git a/api/__init__.py b/api/__init__.py index 583917ec..a75650b7 100644 --- a/api/__init__.py +++ b/api/__init__.py @@ -103,6 +103,7 @@ def create_app(): from api.resources.gene_annotation import gene_annotation from api.resources.interactions import itrns from api.resources.gene_localizations import loc + from api.resources.efp_image import efp_image bar_api.add_namespace(gene_information) bar_api.add_namespace(rnaseq_gene_expression) @@ -115,6 +116,7 @@ def create_app(): bar_api.add_namespace(gene_annotation) bar_api.add_namespace(itrns) bar_api.add_namespace(loc) + bar_api.add_namespace(efp_image) bar_api.init_app(bar_app) return bar_app diff --git a/api/resources/efp_image.py b/api/resources/efp_image.py new file mode 100644 index 00000000..a1c4d752 --- /dev/null +++ b/api/resources/efp_image.py @@ -0,0 +1,15 @@ +from flask_restx import Namespace, Resource +# from markupsafe import escape +from api.utils.bar_utils import BARUtils + +efp_image = Namespace( + "eFP Image", description="eFP Image generation service", path="/efp_image" +) + + +@efp_image.route("/") +class GeneAliasList(Resource): + def get(self): + """This end point returns the list of species available""" + species = ["efp_arabidopsis"] # This are the only species available so far + return BARUtils.success_exit(species) \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 7c26d032..42abbfc7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,9 +5,9 @@ black==20.8b1 certifi==2021.10.8 cffi==1.15.0 chardet==4.0.0 -charset-normalizer==2.0.8 +charset-normalizer==2.0.9 click==8.0.3 -coverage==6.1.2 +coverage==6.2 cryptography==36.0.0 Deprecated==1.2.13 docopt==0.6.2 @@ -15,7 +15,7 @@ flake8==4.0.1 Flask==2.0.2 Flask-Caching==1.10.1 Flask-Cors==3.0.10 -Flask-Limiter==1.4 +Flask-Limiter==2.0.2 flask-marshmallow==0.14.0 flask-restx==0.5.1 Flask-SQLAlchemy==2.5.1 @@ -25,7 +25,7 @@ iniconfig==1.1.1 itsdangerous==2.0.1 Jinja2==3.0.3 jsonschema==4.2.1 -limits==1.5.1 +limits==2.0.3 MarkupSafe==2.0.1 marshmallow==3.14.1 mccabe==0.6.1 @@ -53,8 +53,8 @@ scour==0.38.2 six==1.16.0 SQLAlchemy==1.4.27 toml==0.10.2 -typed-ast==1.5.0 -typing_extensions==4.0.0 +typed-ast==1.5.1 +typing_extensions==4.0.1 urllib3==1.26.7 wcwidth==0.2.5 Werkzeug==2.0.2 diff --git a/tests/resources/test_efp_image.py b/tests/resources/test_efp_image.py new file mode 100644 index 00000000..87c6d0fe --- /dev/null +++ b/tests/resources/test_efp_image.py @@ -0,0 +1,15 @@ +from api import app +from unittest import TestCase + + +class TestIntegrations(TestCase): + def setUp(self): + self.app_client = app.test_client() + + def test_get_efp_image_list(self): + """This function tests the gene alias list get function + :return: + """ + response = self.app_client.get("/efp_image/") + expected = {"wasSuccessful": True, "data": ["efp_arabidopsis"]} + self.assertEqual(response.json, expected) From 142fb5e4e9bbe89aa74d45ee83aa0e4e822704c1 Mon Sep 17 00:00:00 2001 From: asherpasha Date: Sun, 5 Dec 2021 14:54:01 -0500 Subject: [PATCH 12/33] Minor bug. --- api/resources/efp_image.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/resources/efp_image.py b/api/resources/efp_image.py index a1c4d752..bd4e4c4c 100644 --- a/api/resources/efp_image.py +++ b/api/resources/efp_image.py @@ -12,4 +12,4 @@ class GeneAliasList(Resource): def get(self): """This end point returns the list of species available""" species = ["efp_arabidopsis"] # This are the only species available so far - return BARUtils.success_exit(species) \ No newline at end of file + return BARUtils.success_exit(species) From 4af9ac81bda144e2322b2bf4a9bc76ebe7b76434 Mon Sep 17 00:00:00 2001 From: asherpasha Date: Mon, 6 Dec 2021 00:52:15 -0500 Subject: [PATCH 13/33] Working on eFP Images. --- .flake8 | 2 + .gitignore | 1 + api/__init__.py | 4 +- api/resources/api_manager.py | 23 ++++--- api/resources/efp_image.py | 75 +++++++++++++++++++++- api/resources/interactions.py | 5 +- docs/source/conf.py | 18 +++--- tests/resources/test_interactions.py | 49 +++++++-------- tests/resources/test_localizations.py | 91 ++++++++------------------- 9 files changed, 149 insertions(+), 119 deletions(-) diff --git a/.flake8 b/.flake8 index 44ce22a9..6206af19 100644 --- a/.flake8 +++ b/.flake8 @@ -11,4 +11,6 @@ exclude = venv, env, .venv, + ./venv-docs, + docs, .env diff --git a/.gitignore b/.gitignore index ca846006..e61990f0 100644 --- a/.gitignore +++ b/.gitignore @@ -138,3 +138,4 @@ dmypy.json **/.DS_Store .vscode/ +output/* diff --git a/api/__init__.py b/api/__init__.py index a75650b7..6465d283 100644 --- a/api/__init__.py +++ b/api/__init__.py @@ -20,7 +20,9 @@ def create_app(): print("We are now loading configuration.") bar_app.config.from_pyfile(os.getcwd() + "/config/BAR_API.cfg", silent=True) if bar_app.config.get("ADMIN_ENCRYPT_KEY"): - os.environ["ADMIN_ENCRYPT_KEY"] = bar_app.config.get("TEST_ADMIN_ENCRYPT_KEY") + os.environ["ADMIN_ENCRYPT_KEY"] = bar_app.config.get( + "TEST_ADMIN_ENCRYPT_KEY" + ) if bar_app.config.get("ADMIN_PASSWORD_FILE"): os.environ["ADMIN_PASSWORD_FILE"] = bar_app.config.get( "TEST_ADMIN_PASSWORD_FILE" diff --git a/api/resources/api_manager.py b/api/resources/api_manager.py index 1bdd49c1..6be2e8dc 100644 --- a/api/resources/api_manager.py +++ b/api/resources/api_manager.py @@ -174,18 +174,23 @@ def post(self): for row in rows ] df = pandas.DataFrame.from_records([values[0]]) - values_df = pandas.DataFrame(columns=['Gene', - 'Sample', - 'Value']) + values_df = pandas.DataFrame(columns=["Gene", "Sample", "Value"]) con = db.get_engine(bind="summarization") try: df.to_sql("users", con, if_exists="append", index=False) - values_df.to_sql(key, con, index_label='index', - dtype={values_df.index.name: String(42), - 'Gene': String(32), - 'Sample': String(32), - 'Value': Float}, - if_exists="append", index=True) + values_df.to_sql( + key, + con, + index_label="index", + dtype={ + values_df.index.name: String(42), + "Gene": String(32), + "Sample": String(32), + "Value": Float, + }, + if_exists="append", + index=True, + ) el = table.query.filter_by(email=email).one() db.session.delete(el) db.session.commit() diff --git a/api/resources/efp_image.py b/api/resources/efp_image.py index bd4e4c4c..810b6467 100644 --- a/api/resources/efp_image.py +++ b/api/resources/efp_image.py @@ -1,5 +1,8 @@ +import re +import requests from flask_restx import Namespace, Resource -# from markupsafe import escape +from markupsafe import escape +from flask import send_from_directory from api.utils.bar_utils import BARUtils efp_image = Namespace( @@ -8,8 +11,76 @@ @efp_image.route("/") -class GeneAliasList(Resource): +class eFPImageList(Resource): def get(self): """This end point returns the list of species available""" species = ["efp_arabidopsis"] # This are the only species available so far return BARUtils.success_exit(species) + + +@efp_image.route("////") +@efp_image.route( + "/////", + doc=False, +) +class eFPImage(Resource): + @efp_image.param("efp", _in="path", default="efp_arabidopsis") + @efp_image.param("view", _in="path", default="Developmental_Map") + @efp_image.param("mode", _in="path", default="Absolute") + @efp_image.param("gene_1", _in="path", default="At1g01010") + @efp_image.param("gene_2", _in="path", default="At3g27340") + def get(self, efp="", view="", mode="", gene_1="", gene_2=""): + """This end point returns eFP images.""" + # list of allowed eFPs + # See endpoint able + efp_list = ["efp_arabidopsis"] + + # Escape input data + efp = escape(efp) + view = escape(view) + mode = escape(mode) + gene_1 = escape(gene_1) + gene_2 = escape(gene_2) + + # Validate values + if efp not in efp_list: + return BARUtils.error_exit("Invalid eFP"), 400 + + # Add rate limiter + # Validate view + # Validate mode + # Validate gene ids + # Check if request is cached + # If request is not cached, run the search + + # Run eFP + efp_url = ( + "https://bar.utoronto.ca/~asher/python3/" + + efp + + "/cgi-bin/efpWeb.cgi?dataSource=" + + view + + "&mode=" + + mode + + "&primaryGene=" + + gene_1 + + "&secondaryGene=" + + gene_2 + + "&grey_low=None&grey_stddev=None" + ) + efp_html = requests.get(efp_url) + + # Now search for something like Date: Mon, 6 Dec 2021 15:36:08 -0500 Subject: [PATCH 14/33] Working on eFP Image service. --- .gitignore | 1 + api/resources/efp_image.py | 38 ++++++++++++++++++++++++++++++++++---- api/utils/bar_utils.py | 11 +++++++++++ 3 files changed, 46 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index e61990f0..ea420341 100644 --- a/.gitignore +++ b/.gitignore @@ -139,3 +139,4 @@ dmypy.json .vscode/ output/* +!output \ No newline at end of file diff --git a/api/resources/efp_image.py b/api/resources/efp_image.py index 810b6467..1f7c98f5 100644 --- a/api/resources/efp_image.py +++ b/api/resources/efp_image.py @@ -24,6 +24,19 @@ def get(self): doc=False, ) class eFPImage(Resource): + @staticmethod + def is_efp_mode(efp_mode): + """This function checks if the eFP mode is valid + :param efp_mode: string eFP Mode + :return: True or False + """ + # This are case sensitive + valid_modes = ["Absolute", "Relative", "Compare"] + if efp_mode in valid_modes: + return True + else: + return False + @efp_image.param("efp", _in="path", default="efp_arabidopsis") @efp_image.param("view", _in="path", default="Developmental_Map") @efp_image.param("mode", _in="path", default="Absolute") @@ -44,16 +57,28 @@ def get(self, efp="", view="", mode="", gene_1="", gene_2=""): # Validate values if efp not in efp_list: - return BARUtils.error_exit("Invalid eFP"), 400 + return BARUtils.error_exit("Invalid eFP."), 400 - # Add rate limiter # Validate view + if not BARUtils.is_efp_view_name(view): + return BARUtils.error_exit("Invalid eFP View name."), 400 + # Validate mode + if not self.is_efp_mode(mode): + return BARUtils.error_exit("Invalid eFP mode."), 400 + # Validate gene ids + if not BARUtils.is_arabidopsis_gene_valid(gene_1): + return BARUtils.error_exit("Gene 1 is invalid."), 400 + + if mode == "Compare": + if not BARUtils.is_arabidopsis_gene_valid(gene_2): + return BARUtils.error_exit("Gene 2 is invalid."), 400 + # Check if request is cached - # If request is not cached, run the search - # Run eFP + # If request is not cached, run the search + # Run eFP. Note, this is currently running from home directory efp_url = ( "https://bar.utoronto.ca/~asher/python3/" + efp @@ -72,6 +97,11 @@ def get(self, efp="", view="", mode="", gene_1="", gene_2=""): # Now search for something like Date: Mon, 6 Dec 2021 16:09:00 -0500 Subject: [PATCH 15/33] Adding output --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index ea420341..d2661a67 100644 --- a/.gitignore +++ b/.gitignore @@ -139,4 +139,4 @@ dmypy.json .vscode/ output/* -!output \ No newline at end of file +!output From 356724fdbbb396f6cb749d37f964fa16ed707da4 Mon Sep 17 00:00:00 2001 From: asherpasha Date: Mon, 6 Dec 2021 16:11:10 -0500 Subject: [PATCH 16/33] Added output directory. --- .gitignore | 1 + output/placeholder.txt | 1 + 2 files changed, 2 insertions(+) create mode 100644 output/placeholder.txt diff --git a/.gitignore b/.gitignore index d2661a67..7e08bbba 100644 --- a/.gitignore +++ b/.gitignore @@ -140,3 +140,4 @@ dmypy.json .vscode/ output/* !output +!output/placeholder.txt diff --git a/output/placeholder.txt b/output/placeholder.txt new file mode 100644 index 00000000..e118c242 --- /dev/null +++ b/output/placeholder.txt @@ -0,0 +1 @@ +This file exists simply so the output folder will be created when cloning repository. From 955304f337f75bfb7afa85c4a814e4adda6c3fd1 Mon Sep 17 00:00:00 2001 From: asherpasha Date: Wed, 8 Dec 2021 12:57:59 -0500 Subject: [PATCH 17/33] Added unit tests for eFP Image service. --- .github/workflows/bar-api.yml | 2 +- api/resources/efp_image.py | 15 ++++++-- api/utils/bar_utils.py | 2 +- docs/requirements.txt | 2 +- tests/resources/test_efp_image.py | 64 +++++++++++++++++++++++++++++++ tests/utils/test_bar_utils.py | 5 +++ 6 files changed, 83 insertions(+), 7 deletions(-) diff --git a/.github/workflows/bar-api.yml b/.github/workflows/bar-api.yml index df6166a5..5222361f 100644 --- a/.github/workflows/bar-api.yml +++ b/.github/workflows/bar-api.yml @@ -58,4 +58,4 @@ jobs: - name: "Upload coverage to Codecov" uses: codecov/codecov-action@v2 with: - fail_ci_if_error: true + fail_ci_if_error: false diff --git a/api/resources/efp_image.py b/api/resources/efp_image.py index 1f7c98f5..60c760da 100644 --- a/api/resources/efp_image.py +++ b/api/resources/efp_image.py @@ -14,7 +14,9 @@ class eFPImageList(Resource): def get(self): """This end point returns the list of species available""" - species = ["efp_arabidopsis"] # This are the only species available so far + # This are the only species available so far + # If this is updated, update get request and test as well + species = ["efp_arabidopsis"] return BARUtils.success_exit(species) @@ -46,7 +48,7 @@ def get(self, efp="", view="", mode="", gene_1="", gene_2=""): """This end point returns eFP images.""" # list of allowed eFPs # See endpoint able - efp_list = ["efp_arabidopsis"] + species = ["efp_arabidopsis"] # Escape input data efp = escape(efp) @@ -56,7 +58,7 @@ def get(self, efp="", view="", mode="", gene_1="", gene_2=""): gene_2 = escape(gene_2) # Validate values - if efp not in efp_list: + if efp not in species: return BARUtils.error_exit("Invalid eFP."), 400 # Validate view @@ -100,7 +102,12 @@ def get(self, efp="", view="", mode="", gene_1="", gene_2=""): # File is not found if match is None: - return BARUtils.error_exit("Failed to retrieve image. Data for the given gene may not exist."), 500 + return ( + BARUtils.error_exit( + "Failed to retrieve image. Data for the given gene may not exist." + ), + 500, + ) efp_file_link = ( "https://bar.utoronto.ca/~asher/python3/" + efp + "/output/" + match[1] diff --git a/api/utils/bar_utils.py b/api/utils/bar_utils.py index c6861e81..a5a8cea7 100644 --- a/api/utils/bar_utils.py +++ b/api/utils/bar_utils.py @@ -95,7 +95,7 @@ def is_efp_view_name(efp_view): :param efp_view: string view name :return: True if valid """ - if efp_view and re.search(r"[a-z1-9_]{1,20}", efp_view, re.I): + if efp_view and re.search(r"^[a-z1-9_]{1,20}$", efp_view, re.I): return True else: return False diff --git a/docs/requirements.txt b/docs/requirements.txt index eeb1665d..c74e6c05 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -2,7 +2,7 @@ alabaster==0.7.12 Babel==2.9.1 beautifulsoup4==4.10.0 certifi==2021.10.8 -charset-normalizer==2.0.9 +charset-normalizer==2.0.8 docutils==0.17.1 furo==2021.11.23 idna==3.3 diff --git a/tests/resources/test_efp_image.py b/tests/resources/test_efp_image.py index 87c6d0fe..18369385 100644 --- a/tests/resources/test_efp_image.py +++ b/tests/resources/test_efp_image.py @@ -13,3 +13,67 @@ def test_get_efp_image_list(self): response = self.app_client.get("/efp_image/") expected = {"wasSuccessful": True, "data": ["efp_arabidopsis"]} self.assertEqual(response.json, expected) + + def test_get_efp_image(self): + """This function test eFP image endpoint get request + :return: + """ + # A very basic test for Arabidopsis requests + # https://bar.utoronto.ca/api/efp_image/efp_arabidopsis/Developmental_Map/Absolute/At1g01010 + response = self.app_client.get( + "/efp_image/efp_arabidopsis/Developmental_Map/Absolute/At1g01010" + ) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.content_type, "image/png") + self.assertEqual(response.content_length, 190879) + + # A very basic test for Arabidopsis requests + # https://bar.utoronto.ca/api/efp_image/efp_arabidopsis/Developmental_Map/Compare/At1g01010/At1g01030 + response = self.app_client.get( + "/efp_image/efp_arabidopsis/Developmental_Map/Compare/At1g01010/At1g01030" + ) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.content_type, "image/png") + self.assertEqual(response.content_length, 193587) + + # Test for invalid species: + response = self.app_client.get( + "/efp_image/abc/Developmental_Map/Absolute/At1g01010" + ) + expected = {"wasSuccessful": False, "error": "Invalid eFP."} + self.assertEqual(response.json, expected) + + # Test for eFP view name + response = self.app_client.get( + "/efp_image/efp_arabidopsis/ab.!c/Absolute/At1g01010" + ) + expected = {"wasSuccessful": False, "error": "Invalid eFP View name."} + self.assertEqual(response.json, expected) + + # Test for eFP mode + response = self.app_client.get("/efp_image/efp_arabidopsis/Root/abc/At1g01010") + expected = {"wasSuccessful": False, "error": "Invalid eFP mode."} + self.assertEqual(response.json, expected) + + # Test for gene 1 using Arabidopsis + response = self.app_client.get( + "/efp_image/efp_arabidopsis/Root/Absolute/At1g0101X" + ) + expected = {"wasSuccessful": False, "error": "Gene 1 is invalid."} + self.assertEqual(response.json, expected) + + response = self.app_client.get( + "/efp_image/efp_arabidopsis/Developmental_Map/Absolute/At1g01011" + ) + expected = { + "wasSuccessful": False, + "error": "Failed to retrieve image. Data for the given gene may not exist.", + } + self.assertEqual(response.json, expected) + + # Test for gene 2 using Arabidopsis + response = self.app_client.get( + "/efp_image/efp_arabidopsis/Root/Compare/At1g01010/Abc" + ) + expected = {"wasSuccessful": False, "error": "Gene 2 is invalid."} + self.assertEqual(response.json, expected) diff --git a/tests/utils/test_bar_utils.py b/tests/utils/test_bar_utils.py index ac9ab76b..f7d06e64 100644 --- a/tests/utils/test_bar_utils.py +++ b/tests/utils/test_bar_utils.py @@ -28,6 +28,11 @@ def test_is_arabidopsis_gene_valid(self): result = BARUtils.is_arabidopsis_gene_valid("At1g01010.11") self.assertFalse(result) + def test_is_tomato_gene_valid(self): + # For some reason, coverage is saying that we need this test + result = BARUtils.is_tomato_gene_valid("Solyc04g014530") + self.assertTrue(result) + def test_is_integer(self): # Valid result result = BARUtils.is_integer("5") From 0d6d515e38d991493d3f65d2f4b9da211e30f82e Mon Sep 17 00:00:00 2001 From: Bruno Pereira Date: Fri, 10 Dec 2021 12:51:18 -0500 Subject: [PATCH 18/33] Send email notification on request --- api/__init__.py | 12 ++++++++++++ api/resources/api_manager.py | 36 ++++++++++++++++++++++++++++++++++++ config/BAR_API.cfg | 4 ++++ 3 files changed, 52 insertions(+) diff --git a/api/__init__.py b/api/__init__.py index 583917ec..533ac457 100644 --- a/api/__init__.py +++ b/api/__init__.py @@ -28,6 +28,12 @@ def create_app(): elif os.environ.get("BAR"): # The BAR bar_app.config.from_pyfile(os.environ.get("BAR_API_PATH"), silent=True) + if bar_app.config.get("ADMIN_EMAIL"): + os.environ["ADMIN_EMAIL"] = bar_app.config.get("ADMIN_EMAIL") + if bar_app.config.get("EMAIL_PASS_KEY"): + os.environ["EMAIL_PASS_KEY"] = bar_app.config.get("EMAIL_PASS_KEY") + if bar_app.config.get("EMAIL_PASS_FILE"): + os.environ["EMAIL_PASS_FILE"] = bar_app.config.get("EMAIL_PASS_FILE") if bar_app.config.get("ADMIN_ENCRYPT_KEY"): os.environ["ADMIN_ENCRYPT_KEY"] = bar_app.config.get("ADMIN_ENCRYPT_KEY") if bar_app.config.get("ADMIN_PASSWORD_FILE"): @@ -51,6 +57,12 @@ def create_app(): os.environ["ADMIN_PASSWORD_FILE"] = bar_app.config.get( "ADMIN_PASSWORD_FILE" ) + if bar_app.config.get("ADMIN_EMAIL"): + os.environ["ADMIN_EMAIL"] = bar_app.config.get("ADMIN_EMAIL") + if bar_app.config.get("EMAIL_PASS_KEY"): + os.environ["EMAIL_PASS_KEY"] = bar_app.config.get("EMAIL_PASS_KEY") + if bar_app.config.get("EMAIL_PASS_FILE"): + os.environ["EMAIL_PASS_FILE"] = bar_app.config.get("EMAIL_PASS_FILE") if bar_app.config.get("DRIVE_LIST_KEY"): os.environ["DRIVE_LIST_KEY"] = bar_app.config.get("DRIVE_LIST_KEY") if bar_app.config.get("DRIVE_LIST_FILE"): diff --git a/api/resources/api_manager.py b/api/resources/api_manager.py index 1bdd49c1..95de81c2 100644 --- a/api/resources/api_manager.py +++ b/api/resources/api_manager.py @@ -11,6 +11,10 @@ import requests import pandas from cryptography.fernet import Fernet +from smtplib import SMTP_SSL +from ssl import create_default_context +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText CAPTCHA_KEY_FILE = "/home/bpereira/data/bar.summarization/key" @@ -33,6 +37,37 @@ def check_admin_pass(password): else: return False + @staticmethod + def send_email_notification(): + with open(os.environ.get("ADMIN_EMAIL"), "r") as f: + for line in f: + recipient = line + port = 465 + key = os.environ.get('EMAIL_PASS_KEY') + cipher_suite = Fernet(key) + with open(os.environ.get('EMAIL_PASS_FILE'), "rb") as f: + for line in f: + encrypted_key = line + uncipher_text = cipher_suite.decrypt(encrypted_key) + password = bytes(uncipher_text).decode("utf-8") + context = create_default_context() + smtp_server = 'smtp.gmail.com' + sender_email = 'bar.summarization@gmail.com' + subject = "[Bio-Analytic Resource] New API key request" + text = """\ + There is a new API key request. + You can approve or reject it at http://bar.utoronto.ca/~bpereira/webservices/bar-request-manager/build/index.html + """ + m_text = MIMEText(text, _subtype='plain', _charset='UTF-8') + msg = MIMEMultipart() + msg["From"] = sender_email + msg["To"] = recipient + msg["Subject"] = subject + msg.attach(m_text) + with SMTP_SSL(smtp_server, port, context=context) as server: + server.login("bar.summarization@gmail.com", password) + server.sendmail(sender_email, recipient, msg.as_string()) + @api_manager.route("/validate_admin_password", methods=["POST"], doc=False) class ApiManagerValidate(Resource): @@ -84,6 +119,7 @@ def post(self): if row_req is None and row_users is None: df.to_sql("requests", con, if_exists="append", index=False) + ApiManagerUtils.send_email_notification() return BARUtils.success_exit("Data added") else: return BARUtils.error_exit("E-mail already in use"), 409 diff --git a/config/BAR_API.cfg b/config/BAR_API.cfg index 9eb8a74c..140ec1ac 100644 --- a/config/BAR_API.cfg +++ b/config/BAR_API.cfg @@ -23,9 +23,13 @@ SQLALCHEMY_BINDS = { } ## API Manager variables +ADMIN_EMAIL = '/home/bpereira/data/adminemail' ADMIN_PASSWORD_FILE = '/home/bpereira/dev/pw-script/managerkey.bin' ADMIN_ENCRYPT_KEY = 'fabUGnTJ3UQ4qeDJbnSMrb-tDdmt9kxLkuq3GHKdGTo=' +EMAIL_PASS_KEY = '1_2SkFWmTeFnLWtO2oUIRi8pmSd2bMDAtlWq9khpzNc=' +EMAIL_PASS_FILE = '/home/bpereira/dev/pw-script/emailkey.bin' + TEST_ADMIN_PASSWORD_FILE = './tests/data/test_key.bin' TEST_ADMIN_ENCRYPT_KEY = 'fZ2zIiSoV3MOlwiyhjxPWomIArj6YwuDsDNP0J68ZT8=' From 923743be37e0ba2928d32756772e120c2eb35c22 Mon Sep 17 00:00:00 2001 From: Bruno Pereira Date: Fri, 10 Dec 2021 13:13:33 -0500 Subject: [PATCH 19/33] Load env vars in CI --- api/__init__.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/api/__init__.py b/api/__init__.py index 202e33ef..ea91f1d7 100644 --- a/api/__init__.py +++ b/api/__init__.py @@ -27,6 +27,12 @@ def create_app(): os.environ["ADMIN_PASSWORD_FILE"] = bar_app.config.get( "TEST_ADMIN_PASSWORD_FILE" ) + if bar_app.config.get("ADMIN_EMAIL"): + os.environ["ADMIN_EMAIL"] = bar_app.config.get("ADMIN_EMAIL") + if bar_app.config.get("EMAIL_PASS_KEY"): + os.environ["EMAIL_PASS_KEY"] = bar_app.config.get("EMAIL_PASS_KEY") + if bar_app.config.get("EMAIL_PASS_FILE"): + os.environ["EMAIL_PASS_FILE"] = bar_app.config.get("EMAIL_PASS_FILE") elif os.environ.get("BAR"): # The BAR bar_app.config.from_pyfile(os.environ.get("BAR_API_PATH"), silent=True) From fec8e146c4671e4e8e92e870d5f2258a3c6ab331 Mon Sep 17 00:00:00 2001 From: Bruno Pereira Date: Fri, 10 Dec 2021 13:17:59 -0500 Subject: [PATCH 20/33] Ignore email in CI --- api/resources/api_manager.py | 57 ++++++++++++++++++------------------ 1 file changed, 29 insertions(+), 28 deletions(-) diff --git a/api/resources/api_manager.py b/api/resources/api_manager.py index 98d13d88..cdeced3b 100644 --- a/api/resources/api_manager.py +++ b/api/resources/api_manager.py @@ -39,34 +39,35 @@ def check_admin_pass(password): @staticmethod def send_email_notification(): - with open(os.environ.get("ADMIN_EMAIL"), "r") as f: - for line in f: - recipient = line - port = 465 - key = os.environ.get('EMAIL_PASS_KEY') - cipher_suite = Fernet(key) - with open(os.environ.get('EMAIL_PASS_FILE'), "rb") as f: - for line in f: - encrypted_key = line - uncipher_text = cipher_suite.decrypt(encrypted_key) - password = bytes(uncipher_text).decode("utf-8") - context = create_default_context() - smtp_server = 'smtp.gmail.com' - sender_email = 'bar.summarization@gmail.com' - subject = "[Bio-Analytic Resource] New API key request" - text = """\ - There is a new API key request. - You can approve or reject it at http://bar.utoronto.ca/~bpereira/webservices/bar-request-manager/build/index.html - """ - m_text = MIMEText(text, _subtype='plain', _charset='UTF-8') - msg = MIMEMultipart() - msg["From"] = sender_email - msg["To"] = recipient - msg["Subject"] = subject - msg.attach(m_text) - with SMTP_SSL(smtp_server, port, context=context) as server: - server.login("bar.summarization@gmail.com", password) - server.sendmail(sender_email, recipient, msg.as_string()) + if os.environ.get("BAR"): + with open(os.environ.get("ADMIN_EMAIL"), "r") as f: + for line in f: + recipient = line + port = 465 + key = os.environ.get('EMAIL_PASS_KEY') + cipher_suite = Fernet(key) + with open(os.environ.get('EMAIL_PASS_FILE'), "rb") as f: + for line in f: + encrypted_key = line + uncipher_text = cipher_suite.decrypt(encrypted_key) + password = bytes(uncipher_text).decode("utf-8") + context = create_default_context() + smtp_server = 'smtp.gmail.com' + sender_email = 'bar.summarization@gmail.com' + subject = "[Bio-Analytic Resource] New API key request" + text = """\ + There is a new API key request. + You can approve or reject it at http://bar.utoronto.ca/~bpereira/webservices/bar-request-manager/build/index.html + """ + m_text = MIMEText(text, _subtype='plain', _charset='UTF-8') + msg = MIMEMultipart() + msg["From"] = sender_email + msg["To"] = recipient + msg["Subject"] = subject + msg.attach(m_text) + with SMTP_SSL(smtp_server, port, context=context) as server: + server.login("bar.summarization@gmail.com", password) + server.sendmail(sender_email, recipient, msg.as_string()) @api_manager.route("/validate_admin_password", methods=["POST"], doc=False) From 563903accb8812860df03c17d680b5d9683e2892 Mon Sep 17 00:00:00 2001 From: asherpasha Date: Sat, 11 Dec 2021 13:12:48 -0500 Subject: [PATCH 21/33] eFP Image end point is now cached. --- api/resources/api_manager.py | 10 +-- api/resources/efp_image.py | 103 +++++++++++++++++++----------- api/utils/bar_utils.py | 13 ++++ docs/requirements.txt | 4 +- requirements.txt | 2 +- tests/resources/test_efp_image.py | 24 +++++-- 6 files changed, 106 insertions(+), 50 deletions(-) diff --git a/api/resources/api_manager.py b/api/resources/api_manager.py index cdeced3b..2180c9b9 100644 --- a/api/resources/api_manager.py +++ b/api/resources/api_manager.py @@ -44,22 +44,22 @@ def send_email_notification(): for line in f: recipient = line port = 465 - key = os.environ.get('EMAIL_PASS_KEY') + key = os.environ.get("EMAIL_PASS_KEY") cipher_suite = Fernet(key) - with open(os.environ.get('EMAIL_PASS_FILE'), "rb") as f: + with open(os.environ.get("EMAIL_PASS_FILE"), "rb") as f: for line in f: encrypted_key = line uncipher_text = cipher_suite.decrypt(encrypted_key) password = bytes(uncipher_text).decode("utf-8") context = create_default_context() - smtp_server = 'smtp.gmail.com' - sender_email = 'bar.summarization@gmail.com' + smtp_server = "smtp.gmail.com" + sender_email = "bar.summarization@gmail.com" subject = "[Bio-Analytic Resource] New API key request" text = """\ There is a new API key request. You can approve or reject it at http://bar.utoronto.ca/~bpereira/webservices/bar-request-manager/build/index.html """ - m_text = MIMEText(text, _subtype='plain', _charset='UTF-8') + m_text = MIMEText(text, _subtype="plain", _charset="UTF-8") msg = MIMEMultipart() msg["From"] = sender_email msg["To"] = recipient diff --git a/api/resources/efp_image.py b/api/resources/efp_image.py index 60c760da..66b66abf 100644 --- a/api/resources/efp_image.py +++ b/api/resources/efp_image.py @@ -1,5 +1,8 @@ +import base64 import re import requests +import random +import redis.connection from flask_restx import Namespace, Resource from markupsafe import escape from flask import send_from_directory @@ -78,46 +81,72 @@ def get(self, efp="", view="", mode="", gene_1="", gene_2=""): return BARUtils.error_exit("Gene 2 is invalid."), 400 # Check if request is cached - - # If request is not cached, run the search - # Run eFP. Note, this is currently running from home directory - efp_url = ( - "https://bar.utoronto.ca/~asher/python3/" - + efp - + "/cgi-bin/efpWeb.cgi?dataSource=" - + view - + "&mode=" - + mode - + "&primaryGene=" - + gene_1 - + "&secondaryGene=" - + gene_2 - + "&grey_low=None&grey_stddev=None" - ) - efp_html = requests.get(efp_url) - - # Now search for something like Date: Sat, 11 Dec 2021 14:01:17 -0500 Subject: [PATCH 22/33] Fixing read the docs. --- docs/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 954dc840..eeb1665d 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -3,7 +3,7 @@ Babel==2.9.1 beautifulsoup4==4.10.0 certifi==2021.10.8 charset-normalizer==2.0.9 -docutils==0.18.1 +docutils==0.17.1 furo==2021.11.23 idna==3.3 imagesize==1.3.0 From e2529e26cf3400ce9ebe3e4b1f29edf8b577ba0e Mon Sep 17 00:00:00 2001 From: asherpasha Date: Sun, 12 Dec 2021 11:27:07 -0500 Subject: [PATCH 23/33] Adding eFP Cannabis for faster tests. --- api/resources/efp_image.py | 48 ++++----------------- api/utils/bar_utils.py | 22 +++++----- api/utils/efp_utils.py | 69 +++++++++++++++++++++++++++++++ requirements.txt | 2 +- tests/resources/test_efp_image.py | 16 +++---- tests/utils/test_efp_utils.py | 49 ++++++++++++++++++++++ 6 files changed, 146 insertions(+), 60 deletions(-) create mode 100644 api/utils/efp_utils.py create mode 100644 tests/utils/test_efp_utils.py diff --git a/api/resources/efp_image.py b/api/resources/efp_image.py index 66b66abf..d918bb48 100644 --- a/api/resources/efp_image.py +++ b/api/resources/efp_image.py @@ -2,11 +2,12 @@ import re import requests import random -import redis.connection +import redis.exceptions from flask_restx import Namespace, Resource from markupsafe import escape from flask import send_from_directory from api.utils.bar_utils import BARUtils +from api.utils.efp_utils import eFPUtils efp_image = Namespace( "eFP Image", description="eFP Image generation service", path="/efp_image" @@ -18,8 +19,8 @@ class eFPImageList(Resource): def get(self): """This end point returns the list of species available""" # This are the only species available so far - # If this is updated, update get request and test as well - species = ["efp_arabidopsis"] + # If this is updated, update efp_utils.py and unit tests as well + species = ["efp_arabidopsis", "efp_cannabis"] return BARUtils.success_exit(species) @@ -29,19 +30,6 @@ def get(self): doc=False, ) class eFPImage(Resource): - @staticmethod - def is_efp_mode(efp_mode): - """This function checks if the eFP mode is valid - :param efp_mode: string eFP Mode - :return: True or False - """ - # This are case sensitive - valid_modes = ["Absolute", "Relative", "Compare"] - if efp_mode in valid_modes: - return True - else: - return False - @efp_image.param("efp", _in="path", default="efp_arabidopsis") @efp_image.param("view", _in="path", default="Developmental_Map") @efp_image.param("mode", _in="path", default="Absolute") @@ -49,10 +37,6 @@ def is_efp_mode(efp_mode): @efp_image.param("gene_2", _in="path", default="At3g27340") def get(self, efp="", view="", mode="", gene_1="", gene_2=""): """This end point returns eFP images.""" - # list of allowed eFPs - # See endpoint able - species = ["efp_arabidopsis"] - # Escape input data efp = escape(efp) view = escape(view) @@ -60,32 +44,16 @@ def get(self, efp="", view="", mode="", gene_1="", gene_2=""): gene_1 = escape(gene_1) gene_2 = escape(gene_2) - # Validate values - if efp not in species: - return BARUtils.error_exit("Invalid eFP."), 400 - - # Validate view - if not BARUtils.is_efp_view_name(view): - return BARUtils.error_exit("Invalid eFP View name."), 400 - - # Validate mode - if not self.is_efp_mode(mode): - return BARUtils.error_exit("Invalid eFP mode."), 400 - - # Validate gene ids - if not BARUtils.is_arabidopsis_gene_valid(gene_1): - return BARUtils.error_exit("Gene 1 is invalid."), 400 - - if mode == "Compare": - if not BARUtils.is_arabidopsis_gene_valid(gene_2): - return BARUtils.error_exit("Gene 2 is invalid."), 400 + validation = eFPUtils.is_efp_input_valid(efp, view, mode, gene_1, gene_2) + if validation[0] is False: + return BARUtils.error_exit(validation[1]), 400 # Check if request is cached try: r = BARUtils.connect_redis() key = "BAR_API_efp_image_" + "_".join([efp, view, mode, gene_1, gene_2]) efp_image_base64 = r.get(key) - except redis.connection.ConnectionError: + except redis.exceptions.ConnectionError: # Failed redis connection r = None key = None diff --git a/api/utils/bar_utils.py b/api/utils/bar_utils.py index c1fe521b..9144f31e 100644 --- a/api/utils/bar_utils.py +++ b/api/utils/bar_utils.py @@ -72,6 +72,17 @@ def is_tomato_gene_valid(gene, isoform_id=False): else: return False + @staticmethod + def is_cannabis_gene_valid(gene): + """This function verifies if cannabis gene is valid: AGQN03000001 + :param gene: + :return: True if valid + """ + if gene and re.search(r"^AGQN\d{0,10}$", gene, re.I): + return True + else: + return False + @staticmethod def is_integer(data): """Check if the input is at max ten figure number. @@ -91,17 +102,6 @@ def format_poplar(poplar_gene): """ return poplar_gene.translate(str.maketrans("pOTRIg", "PotriG")) - @staticmethod - def is_efp_view_name(efp_view): - """This function is used for validated eFP View names for eFP service - :param efp_view: string view name - :return: True if valid - """ - if efp_view and re.search(r"^[a-z1-9_]{1,20}$", efp_view, re.I): - return True - else: - return False - @staticmethod def connect_redis(): """This function connects to redis diff --git a/api/utils/efp_utils.py b/api/utils/efp_utils.py new file mode 100644 index 00000000..2ed1336a --- /dev/null +++ b/api/utils/efp_utils.py @@ -0,0 +1,69 @@ +import re +from api.utils.bar_utils import BARUtils + + +class eFPUtils: + @staticmethod + def is_efp_view_name(efp_view): + """This function is used for validated eFP View names for eFP service + :param efp_view: string view name + :return: True if valid + """ + if efp_view and re.search(r"^[a-z1-9_]{1,20}$", efp_view, re.I): + return True + else: + return False + + @staticmethod + def is_efp_mode(efp_mode): + """This function checks if the eFP mode is valid + :param efp_mode: string eFP Mode + :return: True or False + """ + # These are case-sensitive + valid_modes = ["Absolute", "Relative", "Compare"] + if efp_mode in valid_modes: + return True + else: + return False + + @staticmethod + def is_efp_input_valid(efp, view, mode, gene_1, gene_2=None): + """Test if eFP input is valid + :return: List with boolean and string + """ + species = ["efp_arabidopsis", "efp_cannabis"] + + # Validate values + if efp not in species: + return False, "Invalid eFP." + + # Validate view + if not eFPUtils.is_efp_view_name(view): + return False, "Invalid eFP View name." + + # Validate mode + if not eFPUtils.is_efp_mode(mode): + return False, "Invalid eFP mode." + + # Maybe this part could be imported + if efp == "efp_arabidopsis": + # Validate gene ids + if not BARUtils.is_arabidopsis_gene_valid(gene_1): + return False, "Gene 1 is invalid." + + if mode == "Compare": + if not BARUtils.is_arabidopsis_gene_valid(gene_2): + return False, "Gene 2 is invalid." + + if efp == "efp_cannabis": + # Validate gene ids + if not BARUtils.is_cannabis_gene_valid(gene_1): + return False, "Gene 1 is invalid." + + if mode == "Compare": + if not BARUtils.is_cannabis_gene_valid(gene_2): + return False, "Gene 2 is invalid." + + # Assuming all check have passed + return True, None diff --git a/requirements.txt b/requirements.txt index bf3c5a25..137859bd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -34,7 +34,7 @@ mypy-extensions==0.4.3 mysqlclient==2.1.0 numpy==1.21.4 packaging==21.3 -pandas==1.3.4 +pandas==1.3.5 pathspec==0.9.0 pluggy==1.0.0 py==1.11.0 diff --git a/tests/resources/test_efp_image.py b/tests/resources/test_efp_image.py index 97bc30a2..e2520dfd 100644 --- a/tests/resources/test_efp_image.py +++ b/tests/resources/test_efp_image.py @@ -11,7 +11,7 @@ def test_get_efp_image_list(self): :return: """ response = self.app_client.get("/efp_image/") - expected = {"wasSuccessful": True, "data": ["efp_arabidopsis"]} + expected = {"wasSuccessful": True, "data": ["efp_arabidopsis", "efp_cannabis"]} self.assertEqual(response.json, expected) def test_get_efp_image(self): @@ -20,16 +20,16 @@ def test_get_efp_image(self): """ # Test absolute modes in the beginning # A very basic test for Arabidopsis requests - # https://bar.utoronto.ca/api/efp_image/efp_arabidopsis/Developmental_Map/Absolute/At1g01010 + # https://bar.utoronto.ca/api/efp_image/efp_cannabis/Cannabis_Atlas/Absolute/AGQN03000001 response = self.app_client.get( - "/efp_image/efp_arabidopsis/Developmental_Map/Absolute/At1g01010" + "/efp_image/efp_cannabis/Cannabis_Atlas/Absolute/AGQN03000001" ) self.assertEqual(response.status_code, 200) self.assertEqual(response.content_type, "image/png") - # Now rerun for Cached requests. A image should be return from the cache + # Now rerun for Cached requests. An image should be return from the cache response = self.app_client.get( - "/efp_image/efp_arabidopsis/Developmental_Map/Absolute/At1g01010" + "/efp_image/efp_cannabis/Cannabis_Atlas/Absolute/AGQN03000001" ) self.assertEqual(response.status_code, 200) self.assertEqual(response.content_type, "image/png") @@ -78,16 +78,16 @@ def test_get_efp_image(self): # Test compare modes in the end # A very basic test for Arabidopsis requests - # https://bar.utoronto.ca/api/efp_image/efp_arabidopsis/Developmental_Map/Compare/At1g01010/At1g01030 + # https://bar.utoronto.ca/api/efp_image/efp_cannabis/Cannabis_Atlas/Compare/AGQN03000001/AGQN03000012 response = self.app_client.get( - "/efp_image/efp_arabidopsis/Developmental_Map/Compare/At1g01010/At1g01030" + "/efp_image/efp_cannabis/Cannabis_Atlas/Compare/AGQN03000001/AGQN03000012" ) self.assertEqual(response.status_code, 200) self.assertEqual(response.content_type, "image/png") # Rerun for cached request. An image should be returned response = self.app_client.get( - "/efp_image/efp_arabidopsis/Developmental_Map/Compare/At1g01010/At1g01030" + "/efp_image/efp_cannabis/Cannabis_Atlas/Compare/AGQN03000001/AGQN03000012" ) self.assertEqual(response.status_code, 200) self.assertEqual(response.content_type, "image/png") diff --git a/tests/utils/test_efp_utils.py b/tests/utils/test_efp_utils.py new file mode 100644 index 00000000..b777a4eb --- /dev/null +++ b/tests/utils/test_efp_utils.py @@ -0,0 +1,49 @@ +from unittest import TestCase +from api.utils.efp_utils import eFPUtils + + +class UtilsUnitTest(TestCase): + def test_is_efp_input_valid(self): + """Tests for eFP input data""" + + # eFP Arabidopsis compare mode + result = eFPUtils.is_efp_input_valid( + "efp_arabidopsis", "Root", "Compare", "At1g01010", "At1g01030" + ) + self.assertTrue(result[0]) + self.assertIsNone(result[1]) + + result = eFPUtils.is_efp_input_valid( + "efp_arabidopsis", "Root", "Compare", "At1g01010", "Abc" + ) + expected = "Gene 2 is invalid." + self.assertFalse(result[0]) + self.assertEqual(result[1], expected) + + # eFP Cannabis gene1 + result = eFPUtils.is_efp_input_valid( + "efp_cannabis", "Cannabis_Atlas", "Absolute", "AGQN03000001" + ) + self.assertTrue(result[0]) + self.assertIsNone(result[1]) + + result = eFPUtils.is_efp_input_valid( + "efp_cannabis", "Cannabis_Atlas", "Absolute", "Abc" + ) + expected = "Gene 1 is invalid." + self.assertFalse(result[0]) + self.assertEqual(result[1], expected) + + # eFP Cannabis gene2 + result = eFPUtils.is_efp_input_valid( + "efp_cannabis", "Cannabis_Atlas", "Compare", "AGQN03000001", "AGQN03000012" + ) + self.assertTrue(result[0]) + self.assertIsNone(result[1]) + + result = eFPUtils.is_efp_input_valid( + "efp_cannabis", "Cannabis_Atlas", "Compare", "AGQN03000001", "Abc" + ) + expected = "Gene 2 is invalid." + self.assertFalse(result[0]) + self.assertEqual(result[1], expected) From 1cb58727a4df1cebef38155babf00fb5503d20b8 Mon Sep 17 00:00:00 2001 From: asherpasha Date: Sun, 12 Dec 2021 11:50:25 -0500 Subject: [PATCH 24/33] Removing eFP compare test to speed up testing. --- tests/resources/test_efp_image.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/tests/resources/test_efp_image.py b/tests/resources/test_efp_image.py index e2520dfd..23ae3809 100644 --- a/tests/resources/test_efp_image.py +++ b/tests/resources/test_efp_image.py @@ -75,19 +75,3 @@ def test_get_efp_image(self): ) expected = {"wasSuccessful": False, "error": "Gene 2 is invalid."} self.assertEqual(response.json, expected) - - # Test compare modes in the end - # A very basic test for Arabidopsis requests - # https://bar.utoronto.ca/api/efp_image/efp_cannabis/Cannabis_Atlas/Compare/AGQN03000001/AGQN03000012 - response = self.app_client.get( - "/efp_image/efp_cannabis/Cannabis_Atlas/Compare/AGQN03000001/AGQN03000012" - ) - self.assertEqual(response.status_code, 200) - self.assertEqual(response.content_type, "image/png") - - # Rerun for cached request. An image should be returned - response = self.app_client.get( - "/efp_image/efp_cannabis/Cannabis_Atlas/Compare/AGQN03000001/AGQN03000012" - ) - self.assertEqual(response.status_code, 200) - self.assertEqual(response.content_type, "image/png") From 4cccd50712e48b83b950034609f9b140cc31cde5 Mon Sep 17 00:00:00 2001 From: asherpasha Date: Wed, 15 Dec 2021 11:26:09 -0500 Subject: [PATCH 25/33] eFP Arachis image service is added. --- api/resources/efp_image.py | 2 +- api/utils/bar_utils.py | 13 +++++++++ api/utils/efp_utils.py | 15 ++++++++++- requirements.txt | 2 +- tests/resources/test_efp_image.py | 7 +++-- tests/utils/test_efp_utils.py | 44 +++++++++++++++++++++++++++++++ 6 files changed, 78 insertions(+), 5 deletions(-) diff --git a/api/resources/efp_image.py b/api/resources/efp_image.py index d918bb48..d395f9f1 100644 --- a/api/resources/efp_image.py +++ b/api/resources/efp_image.py @@ -20,7 +20,7 @@ def get(self): """This end point returns the list of species available""" # This are the only species available so far # If this is updated, update efp_utils.py and unit tests as well - species = ["efp_arabidopsis", "efp_cannabis"] + species = ["efp_arabidopsis", "efp_cannabis", "efp_arachis"] return BARUtils.success_exit(species) diff --git a/api/utils/bar_utils.py b/api/utils/bar_utils.py index 9144f31e..b0396ec8 100644 --- a/api/utils/bar_utils.py +++ b/api/utils/bar_utils.py @@ -83,6 +83,19 @@ def is_cannabis_gene_valid(gene): else: return False + @staticmethod + def is_arachis_gene_valid(gene): + """This function verifies if arachis gene is valid: Adur10000_comp0_c0_seq1 + :param gene: + :return: True if valid + """ + if gene and re.search( + r"Adur\d{1,10}_comp\d{1,3}_\D{1,3}\d{1,3}_seq\d{1,5}", gene, re.I + ): + return True + else: + return False + @staticmethod def is_integer(data): """Check if the input is at max ten figure number. diff --git a/api/utils/efp_utils.py b/api/utils/efp_utils.py index 2ed1336a..6dbdb1c9 100644 --- a/api/utils/efp_utils.py +++ b/api/utils/efp_utils.py @@ -32,7 +32,7 @@ def is_efp_input_valid(efp, view, mode, gene_1, gene_2=None): """Test if eFP input is valid :return: List with boolean and string """ - species = ["efp_arabidopsis", "efp_cannabis"] + species = ["efp_arabidopsis", "efp_cannabis", "efp_arachis"] # Validate values if efp not in species: @@ -65,5 +65,18 @@ def is_efp_input_valid(efp, view, mode, gene_1, gene_2=None): if not BARUtils.is_cannabis_gene_valid(gene_2): return False, "Gene 2 is invalid." + if efp == "efp_arachis": + # Validate gene ids + if not BARUtils.is_arachis_gene_valid(gene_1): + return False, "Gene 1 is invalid." + + if mode == "Compare": + if not BARUtils.is_arachis_gene_valid(gene_2): + return False, "Gene 2 is invalid." + + # In compare mode gene1 != gene2 + if mode == "Compare" and gene_1 == gene_2: + return False, "In compare mode, both genes should be different." + # Assuming all check have passed return True, None diff --git a/requirements.txt b/requirements.txt index 137859bd..5f96e9cb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,7 @@ chardet==4.0.0 charset-normalizer==2.0.9 click==8.0.3 coverage==6.2 -cryptography==36.0.0 +cryptography==36.0.1 Deprecated==1.2.13 docopt==0.6.2 flake8==4.0.1 diff --git a/tests/resources/test_efp_image.py b/tests/resources/test_efp_image.py index 23ae3809..fc36ad45 100644 --- a/tests/resources/test_efp_image.py +++ b/tests/resources/test_efp_image.py @@ -11,7 +11,10 @@ def test_get_efp_image_list(self): :return: """ response = self.app_client.get("/efp_image/") - expected = {"wasSuccessful": True, "data": ["efp_arabidopsis", "efp_cannabis"]} + expected = { + "wasSuccessful": True, + "data": ["efp_arabidopsis", "efp_cannabis", "efp_arachis"], + } self.assertEqual(response.json, expected) def test_get_efp_image(self): @@ -69,7 +72,7 @@ def test_get_efp_image(self): } self.assertEqual(response.json, expected) - # Test for gene 2 using Arabidopsis + # Test for gene 1 using Arabidopsis response = self.app_client.get( "/efp_image/efp_arabidopsis/Root/Compare/At1g01010/Abc" ) diff --git a/tests/utils/test_efp_utils.py b/tests/utils/test_efp_utils.py index b777a4eb..1ddd5b7d 100644 --- a/tests/utils/test_efp_utils.py +++ b/tests/utils/test_efp_utils.py @@ -47,3 +47,47 @@ def test_is_efp_input_valid(self): expected = "Gene 2 is invalid." self.assertFalse(result[0]) self.assertEqual(result[1], expected) + + # eFP Arachis Absolute + result = eFPUtils.is_efp_input_valid( + "efp_arachis", "Arachis_Atlas", "Absolute", "Adur10002_comp0_c0_seq1" + ) + self.assertTrue(result[0]) + self.assertIsNone(result[1]) + + result = eFPUtils.is_efp_input_valid( + "efp_arachis", "Arachis_Atlas", "Absolute", "Abc" + ) + expected = "Gene 1 is invalid." + self.assertFalse(result[0]) + self.assertEqual(result[1], expected) + + # eFP Arachis gene 2 + result = eFPUtils.is_efp_input_valid( + "efp_arachis", + "Arachis_Atlas", + "Compare", + "Adur10002_comp0_c0_seq1", + "Adur10002_comp0_c0_seq11", + ) + self.assertTrue(result[0]) + self.assertIsNone(result[1]) + + result = eFPUtils.is_efp_input_valid( + "efp_arachis", "Arachis_Atlas", "Compare", "Adur10002_comp0_c0_seq1", "Abc" + ) + expected = "Gene 2 is invalid." + self.assertFalse(result[0]) + self.assertEqual(result[1], expected) + + # Test if both gene are the same in eFP Comare mode + result = eFPUtils.is_efp_input_valid( + "efp_arachis", + "Arachis_Atlas", + "Compare", + "Adur10002_comp0_c0_seq1", + "Adur10002_comp0_c0_seq1", + ) + expected = "In compare mode, both genes should be different." + self.assertFalse(result[0]) + self.assertEqual(result[1], expected) From 874c7f7d7359899cf7ebd0fa8cd6ed6ee3555d17 Mon Sep 17 00:00:00 2001 From: Bruno Pereira Date: Wed, 15 Dec 2021 20:06:49 -0500 Subject: [PATCH 26/33] Validate captcha within /request --- api/resources/api_manager.py | 74 ++++++++++++++++++------------------ 1 file changed, 38 insertions(+), 36 deletions(-) diff --git a/api/resources/api_manager.py b/api/resources/api_manager.py index cdeced3b..a6fad6e2 100644 --- a/api/resources/api_manager.py +++ b/api/resources/api_manager.py @@ -37,6 +37,24 @@ def check_admin_pass(password): else: return False + @staticmethod + def validate_captcha(self): + """Validates a reCaptcha value using our secret token""" + if request.method == "POST": + json = request.get_json() + value = json["response"] + with open(CAPTCHA_KEY_FILE, "rb") as f: + for line in f: + key = line + if key: + ret = requests.post( + "https://www.google.com/recaptcha/api/siteverify", + data={"secret": key, "response": value} + ) + return ret.json()["success"] + else: + return False + @staticmethod def send_email_notification(): if os.environ.get("BAR"): @@ -109,23 +127,27 @@ def post(self): class ApiManagerRequest(Resource): def post(self): if request.method == "POST": - response_json = request.get_json() - df = pandas.DataFrame.from_records([response_json]) - con = db.get_engine(bind="summarization") - try: - reqs = Requests() - users = Users() - row_req = reqs.query.filter_by(email=df.email[0]).first() - row_users = users.query.filter_by(email=df.email[0]).first() + captchaVal = request.headers.get("captchaVal") + if(ApiManagerUtils.validate_captcha(captchaVal)): + response_json = request.get_json() + df = pandas.DataFrame.from_records([response_json]) + con = db.get_engine(bind="summarization") + try: + reqs = Requests() + users = Users() + row_req = reqs.query.filter_by(email=df.email[0]).first() + row_users = users.query.filter_by(email=df.email[0]).first() - if row_req is None and row_users is None: - df.to_sql("requests", con, if_exists="append", index=False) - ApiManagerUtils.send_email_notification() - return BARUtils.success_exit("Data added") - else: - return BARUtils.error_exit("E-mail already in use"), 409 - except SQLAlchemyError: - return BARUtils.error_exit("Internal server error"), 500 + if row_req is None and row_users is None: + df.to_sql("requests", con, if_exists="append", index=False) + ApiManagerUtils.send_email_notification() + return BARUtils.success_exit("Data added") + else: + return BARUtils.error_exit("E-mail already in use"), 409 + except SQLAlchemyError: + return BARUtils.error_exit("Internal server error"), 500 + else: + return BARUtils.error_exit("Failed Captcha verification") @api_manager.route("/get_pending_requests", methods=["POST"], doc=False) @@ -236,23 +258,3 @@ def post(self): return BARUtils.success_exit(key) else: return BARUtils.error_exit("Forbidden"), 403 - - -@api_manager.route("/validate_captcha", methods=["POST"], doc=False) -class ApiManagerCaptchaValidate(Resource): - def post(self): - """Validates a reCaptcha value using our secret token""" - if request.method == "POST": - json = request.get_json() - value = json["response"] - with open(CAPTCHA_KEY_FILE, "rb") as f: - for line in f: - key = line - if key: - ret = requests.post( - "https://www.google.com/recaptcha/api/siteverify", - data={"secret": key, "response": value}, - ) - return BARUtils.success_exit(ret.text) - else: - return BARUtils.error_exit("Forbidden: CAPTCHA key is not found"), 403 From 5d36229767f1afb6d14a98eeec0e26a50f2eb0de Mon Sep 17 00:00:00 2001 From: Bruno Pereira Date: Thu, 16 Dec 2021 17:52:55 -0500 Subject: [PATCH 27/33] Validate Captcha inside /request --- api/resources/api_manager.py | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/api/resources/api_manager.py b/api/resources/api_manager.py index a6fad6e2..8161e77e 100644 --- a/api/resources/api_manager.py +++ b/api/resources/api_manager.py @@ -38,22 +38,19 @@ def check_admin_pass(password): return False @staticmethod - def validate_captcha(self): + def validate_captcha(value): """Validates a reCaptcha value using our secret token""" - if request.method == "POST": - json = request.get_json() - value = json["response"] - with open(CAPTCHA_KEY_FILE, "rb") as f: - for line in f: - key = line - if key: - ret = requests.post( - "https://www.google.com/recaptcha/api/siteverify", - data={"secret": key, "response": value} - ) - return ret.json()["success"] - else: - return False + with open(CAPTCHA_KEY_FILE, "rb") as f: + for line in f: + key = line + if key: + ret = requests.post( + "https://www.google.com/recaptcha/api/siteverify", + data={"secret": key, "response": value} + ) + return ret.json()["success"] + else: + return False @staticmethod def send_email_notification(): From e3841c8e1eaf1b5c3fc3189bf9f070150228e092 Mon Sep 17 00:00:00 2001 From: Bruno Pereira Date: Fri, 17 Dec 2021 13:10:57 -0500 Subject: [PATCH 28/33] Don't try to validate captcha in CI --- api/resources/api_manager.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/api/resources/api_manager.py b/api/resources/api_manager.py index 8161e77e..83b7357f 100644 --- a/api/resources/api_manager.py +++ b/api/resources/api_manager.py @@ -40,17 +40,20 @@ def check_admin_pass(password): @staticmethod def validate_captcha(value): """Validates a reCaptcha value using our secret token""" - with open(CAPTCHA_KEY_FILE, "rb") as f: - for line in f: - key = line - if key: - ret = requests.post( - "https://www.google.com/recaptcha/api/siteverify", - data={"secret": key, "response": value} - ) - return ret.json()["success"] + if(os.environ.get("BAR")): + with open(CAPTCHA_KEY_FILE, "rb") as f: + for line in f: + key = line + if key: + ret = requests.post( + "https://www.google.com/recaptcha/api/siteverify", + data={"secret": key, "response": value} + ) + return ret.json()["success"] + else: + return False else: - return False + return True @staticmethod def send_email_notification(): From 6cc7f6952470c3d2f1967d54cc7d9feab405839f Mon Sep 17 00:00:00 2001 From: asherpasha Date: Fri, 17 Dec 2021 17:35:33 -0500 Subject: [PATCH 29/33] Added eFP Soybean. --- api/resources/efp_image.py | 2 +- api/utils/bar_utils.py | 13 +++++++++++++ api/utils/efp_utils.py | 13 +++++++++++-- requirements.txt | 4 ++-- tests/resources/test_efp_image.py | 2 +- tests/utils/test_efp_utils.py | 32 +++++++++++++++++++++++++++++++ 6 files changed, 60 insertions(+), 6 deletions(-) diff --git a/api/resources/efp_image.py b/api/resources/efp_image.py index d395f9f1..00ea5719 100644 --- a/api/resources/efp_image.py +++ b/api/resources/efp_image.py @@ -20,7 +20,7 @@ def get(self): """This end point returns the list of species available""" # This are the only species available so far # If this is updated, update efp_utils.py and unit tests as well - species = ["efp_arabidopsis", "efp_cannabis", "efp_arachis"] + species = ["efp_arabidopsis", "efp_cannabis", "efp_arachis", "efp_soybean"] return BARUtils.success_exit(species) diff --git a/api/utils/bar_utils.py b/api/utils/bar_utils.py index b0396ec8..6546c8db 100644 --- a/api/utils/bar_utils.py +++ b/api/utils/bar_utils.py @@ -96,6 +96,19 @@ def is_arachis_gene_valid(gene): else: return False + @staticmethod + def is_soybean_gene_valid(gene): + """This function verifies if soybean gene is valid: Glyma06g47400 + :param gene: + :return: True if valid + """ + if gene and re.search( + r"^((Glyma\d{1,3}g\d{1,6}\.?\d?)|(Glyma\.\d{1,3}g\d{1,8}))$", gene, re.I + ): + return True + else: + return False + @staticmethod def is_integer(data): """Check if the input is at max ten figure number. diff --git a/api/utils/efp_utils.py b/api/utils/efp_utils.py index 6dbdb1c9..6ee79585 100644 --- a/api/utils/efp_utils.py +++ b/api/utils/efp_utils.py @@ -9,7 +9,7 @@ def is_efp_view_name(efp_view): :param efp_view: string view name :return: True if valid """ - if efp_view and re.search(r"^[a-z1-9_]{1,20}$", efp_view, re.I): + if efp_view and re.search(r"^[a-z1-9_]{1,50}$", efp_view, re.I): return True else: return False @@ -32,7 +32,7 @@ def is_efp_input_valid(efp, view, mode, gene_1, gene_2=None): """Test if eFP input is valid :return: List with boolean and string """ - species = ["efp_arabidopsis", "efp_cannabis", "efp_arachis"] + species = ["efp_arabidopsis", "efp_cannabis", "efp_arachis", "efp_soybean"] # Validate values if efp not in species: @@ -74,6 +74,15 @@ def is_efp_input_valid(efp, view, mode, gene_1, gene_2=None): if not BARUtils.is_arachis_gene_valid(gene_2): return False, "Gene 2 is invalid." + if efp == "efp_soybean": + # Validate gene ids + if not BARUtils.is_soybean_gene_valid(gene_1): + return False, "Gene 1 is invalid." + + if mode == "Compare": + if not BARUtils.is_soybean_gene_valid(gene_2): + return False, "Gene 2 is invalid." + # In compare mode gene1 != gene2 if mode == "Compare" and gene_1 == gene_2: return False, "In compare mode, both genes should be different." diff --git a/requirements.txt b/requirements.txt index 5f96e9cb..a7c8e524 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,7 +15,7 @@ flake8==4.0.1 Flask==2.0.2 Flask-Caching==1.10.1 Flask-Cors==3.0.10 -Flask-Limiter==2.0.2 +Flask-Limiter==2.0.3 flask-marshmallow==0.14.0 flask-restx==0.5.1 Flask-SQLAlchemy==2.5.1 @@ -24,7 +24,7 @@ idna==3.3 iniconfig==1.1.1 itsdangerous==2.0.1 Jinja2==3.0.3 -jsonschema==4.2.1 +jsonschema==4.3.1 limits==2.0.3 MarkupSafe==2.0.1 marshmallow==3.14.1 diff --git a/tests/resources/test_efp_image.py b/tests/resources/test_efp_image.py index fc36ad45..6850ad86 100644 --- a/tests/resources/test_efp_image.py +++ b/tests/resources/test_efp_image.py @@ -13,7 +13,7 @@ def test_get_efp_image_list(self): response = self.app_client.get("/efp_image/") expected = { "wasSuccessful": True, - "data": ["efp_arabidopsis", "efp_cannabis", "efp_arachis"], + "data": ["efp_arabidopsis", "efp_cannabis", "efp_arachis", "efp_soybean"], } self.assertEqual(response.json, expected) diff --git a/tests/utils/test_efp_utils.py b/tests/utils/test_efp_utils.py index 1ddd5b7d..d64161dd 100644 --- a/tests/utils/test_efp_utils.py +++ b/tests/utils/test_efp_utils.py @@ -80,6 +80,38 @@ def test_is_efp_input_valid(self): self.assertFalse(result[0]) self.assertEqual(result[1], expected) + # eFP Soybean Absolute + result = eFPUtils.is_efp_input_valid( + "efp_soybean", "soybean", "Absolute", "Glyma06g47400" + ) + self.assertTrue(result[0]) + self.assertIsNone(result[1]) + + result = eFPUtils.is_efp_input_valid( + "efp_soybean", "soybean", "Absolute", "Abc" + ) + expected = "Gene 1 is invalid." + self.assertFalse(result[0]) + self.assertEqual(result[1], expected) + + # eFP Soybean gene 2 + result = eFPUtils.is_efp_input_valid( + "efp_soybean", + "soybean", + "Compare", + "Glyma06g47400", + "Glyma06g47390", + ) + self.assertTrue(result[0]) + self.assertIsNone(result[1]) + + result = eFPUtils.is_efp_input_valid( + "efp_soybean", "soybean", "Compare", "Glyma06g47400", "Abc" + ) + expected = "Gene 2 is invalid." + self.assertFalse(result[0]) + self.assertEqual(result[1], expected) + # Test if both gene are the same in eFP Comare mode result = eFPUtils.is_efp_input_valid( "efp_arachis", From 7553f4f17569ae443a32dff7a605e15e0fd75111 Mon Sep 17 00:00:00 2001 From: asherpasha Date: Thu, 30 Dec 2021 11:10:31 -0500 Subject: [PATCH 30/33] eFP Maize is added. Output directory cleanup is also added. --- api/resources/api_manager.py | 6 ++--- api/resources/efp_image.py | 23 +++++++++++++++- api/utils/bar_utils.py | 15 +++++++++++ api/utils/efp_utils.py | 19 +++++++++++-- requirements.txt | 14 +++++----- tests/resources/test_efp_image.py | 8 +++++- tests/utils/test_efp_utils.py | 44 +++++++++++++++++++++++++++++++ 7 files changed, 115 insertions(+), 14 deletions(-) diff --git a/api/resources/api_manager.py b/api/resources/api_manager.py index 1d99c8ac..0a7327ca 100644 --- a/api/resources/api_manager.py +++ b/api/resources/api_manager.py @@ -40,14 +40,14 @@ def check_admin_pass(password): @staticmethod def validate_captcha(value): """Validates a reCaptcha value using our secret token""" - if(os.environ.get("BAR")): + if os.environ.get("BAR"): with open(CAPTCHA_KEY_FILE, "rb") as f: for line in f: key = line if key: ret = requests.post( "https://www.google.com/recaptcha/api/siteverify", - data={"secret": key, "response": value} + data={"secret": key, "response": value}, ) return ret.json()["success"] else: @@ -128,7 +128,7 @@ class ApiManagerRequest(Resource): def post(self): if request.method == "POST": captchaVal = request.headers.get("captchaVal") - if(ApiManagerUtils.validate_captcha(captchaVal)): + if ApiManagerUtils.validate_captcha(captchaVal): response_json = request.get_json() df = pandas.DataFrame.from_records([response_json]) con = db.get_engine(bind="summarization") diff --git a/api/resources/efp_image.py b/api/resources/efp_image.py index 00ea5719..f86e3286 100644 --- a/api/resources/efp_image.py +++ b/api/resources/efp_image.py @@ -2,6 +2,8 @@ import re import requests import random +import os +import time import redis.exceptions from flask_restx import Namespace, Resource from markupsafe import escape @@ -20,7 +22,13 @@ def get(self): """This end point returns the list of species available""" # This are the only species available so far # If this is updated, update efp_utils.py and unit tests as well - species = ["efp_arabidopsis", "efp_cannabis", "efp_arachis", "efp_soybean"] + species = [ + "efp_arabidopsis", + "efp_cannabis", + "efp_arachis", + "efp_soybean", + "efp_maize", + ] return BARUtils.success_exit(species) @@ -48,6 +56,19 @@ def get(self, efp="", view="", mode="", gene_1="", gene_2=""): if validation[0] is False: return BARUtils.error_exit(validation[1]), 400 + # Data are valid. Clear directory before running + for file in os.listdir("output"): + # Full file name is required at this point + file = os.path.join("output", file) + + # Check if it is a png file and is greater than 5 minutes old + if ( + os.path.isfile(file) + and (os.path.splitext(file)[1] == ".png") + and (os.stat(file).st_mtime < (time.time() - 5 * 60)) + ): + os.remove(file) + # Check if request is cached try: r = BARUtils.connect_redis() diff --git a/api/utils/bar_utils.py b/api/utils/bar_utils.py index 6546c8db..80d02d2d 100644 --- a/api/utils/bar_utils.py +++ b/api/utils/bar_utils.py @@ -109,6 +109,21 @@ def is_soybean_gene_valid(gene): else: return False + @staticmethod + def is_maize_gene_valid(gene): + """This function verifies if maize gene is valid: Zm00001d046170 + :param gene: + :return: True if valid + """ + if gene and re.search( + r"^(AC[0-9]{6}\.[0-9]{1}_FG[0-9]{3})|(AC[0-9]{6}\.[0-9]{1}_FGT[0-9]{3})|(GRMZM(2|5)G[0-9]{6})|(GRMZM(2|5)G[0-9]{6}_T[0-9]{2})|(Zm\d+d\d+)$", + gene, + re.I, + ): + return True + else: + return False + @staticmethod def is_integer(data): """Check if the input is at max ten figure number. diff --git a/api/utils/efp_utils.py b/api/utils/efp_utils.py index 6ee79585..b78d1522 100644 --- a/api/utils/efp_utils.py +++ b/api/utils/efp_utils.py @@ -32,7 +32,13 @@ def is_efp_input_valid(efp, view, mode, gene_1, gene_2=None): """Test if eFP input is valid :return: List with boolean and string """ - species = ["efp_arabidopsis", "efp_cannabis", "efp_arachis", "efp_soybean"] + species = [ + "efp_arabidopsis", + "efp_cannabis", + "efp_arachis", + "efp_soybean", + "efp_maize", + ] # Validate values if efp not in species: @@ -46,7 +52,7 @@ def is_efp_input_valid(efp, view, mode, gene_1, gene_2=None): if not eFPUtils.is_efp_mode(mode): return False, "Invalid eFP mode." - # Maybe this part could be imported + # Maybe this part could be improved if efp == "efp_arabidopsis": # Validate gene ids if not BARUtils.is_arabidopsis_gene_valid(gene_1): @@ -83,6 +89,15 @@ def is_efp_input_valid(efp, view, mode, gene_1, gene_2=None): if not BARUtils.is_soybean_gene_valid(gene_2): return False, "Gene 2 is invalid." + if efp == "efp_maize": + # Validate gene ids + if not BARUtils.is_maize_gene_valid(gene_1): + return False, "Gene 1 is invalid." + + if mode == "Compare": + if not BARUtils.is_maize_gene_valid(gene_2): + return False, "Gene 2 is invalid." + # In compare mode gene1 != gene2 if mode == "Compare" and gene_1 == gene_2: return False, "In compare mode, both genes should be different." diff --git a/requirements.txt b/requirements.txt index a7c8e524..f6e5910e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ aniso8601==9.0.1 appdirs==1.4.4 -attrs==21.2.0 +attrs==21.4.0 black==20.8b1 certifi==2021.10.8 cffi==1.15.0 @@ -15,7 +15,7 @@ flake8==4.0.1 Flask==2.0.2 Flask-Caching==1.10.1 Flask-Cors==3.0.10 -Flask-Limiter==2.0.3 +Flask-Limiter==2.0.4 flask-marshmallow==0.14.0 flask-restx==0.5.1 Flask-SQLAlchemy==2.5.1 @@ -24,15 +24,15 @@ idna==3.3 iniconfig==1.1.1 itsdangerous==2.0.1 Jinja2==3.0.3 -jsonschema==4.3.1 -limits==2.0.3 +jsonschema==4.3.2 +limits==2.1.0 MarkupSafe==2.0.1 marshmallow==3.14.1 mccabe==0.6.1 more-itertools==8.12.0 mypy-extensions==0.4.3 mysqlclient==2.1.0 -numpy==1.21.4 +numpy==1.21.5 packaging==21.3 pandas==1.3.5 pathspec==0.9.0 @@ -46,12 +46,12 @@ pyrsistent==0.18.0 pytest==6.2.5 python-dateutil==2.8.2 pytz==2021.3 -redis==4.0.2 +redis==4.1.0 regex==2021.11.10 requests==2.26.0 scour==0.38.2 six==1.16.0 -SQLAlchemy==1.4.28 +SQLAlchemy==1.4.29 toml==0.10.2 typed-ast==1.5.1 typing_extensions==4.0.1 diff --git a/tests/resources/test_efp_image.py b/tests/resources/test_efp_image.py index 6850ad86..0b77e550 100644 --- a/tests/resources/test_efp_image.py +++ b/tests/resources/test_efp_image.py @@ -13,7 +13,13 @@ def test_get_efp_image_list(self): response = self.app_client.get("/efp_image/") expected = { "wasSuccessful": True, - "data": ["efp_arabidopsis", "efp_cannabis", "efp_arachis", "efp_soybean"], + "data": [ + "efp_arabidopsis", + "efp_cannabis", + "efp_arachis", + "efp_soybean", + "efp_maize", + ], } self.assertEqual(response.json, expected) diff --git a/tests/utils/test_efp_utils.py b/tests/utils/test_efp_utils.py index d64161dd..d3bc1a7a 100644 --- a/tests/utils/test_efp_utils.py +++ b/tests/utils/test_efp_utils.py @@ -123,3 +123,47 @@ def test_is_efp_input_valid(self): expected = "In compare mode, both genes should be different." self.assertFalse(result[0]) self.assertEqual(result[1], expected) + + # eFP Maize Absolute + result = eFPUtils.is_efp_input_valid( + "efp_maize", "maize_iplant", "Absolute", "Zm00001d046170" + ) + self.assertTrue(result[0]) + self.assertIsNone(result[1]) + + result = eFPUtils.is_efp_input_valid( + "efp_maize", "maize_iplant", "Absolute", "Abc" + ) + expected = "Gene 1 is invalid." + self.assertFalse(result[0]) + self.assertEqual(result[1], expected) + + # eFP Maize gene 2 + result = eFPUtils.is_efp_input_valid( + "efp_maize", + "maize_iplant", + "Compare", + "Zm00001d046170", + "Zm00001d014297", + ) + self.assertTrue(result[0]) + self.assertIsNone(result[1]) + + result = eFPUtils.is_efp_input_valid( + "efp_maize", "maize_iplant", "Compare", "Zm00001d046170", "Abc" + ) + expected = "Gene 2 is invalid." + self.assertFalse(result[0]) + self.assertEqual(result[1], expected) + + # Test if both gene are the same in eFP Comare mode + result = eFPUtils.is_efp_input_valid( + "efp_arachis", + "Arachis_Atlas", + "Compare", + "Adur10002_comp0_c0_seq1", + "Adur10002_comp0_c0_seq1", + ) + expected = "In compare mode, both genes should be different." + self.assertFalse(result[0]) + self.assertEqual(result[1], expected) From f6ee0c3bdfcbc0c322b90afff311a9c48f3007b7 Mon Sep 17 00:00:00 2001 From: Bruno Pereira Date: Thu, 13 Jan 2022 18:42:41 -0500 Subject: [PATCH 31/33] Fix CSV/TSV saving to use temp directory --- api/resources/summarization_gene_expression.py | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/api/resources/summarization_gene_expression.py b/api/resources/summarization_gene_expression.py index 2d448459..5502a538 100644 --- a/api/resources/summarization_gene_expression.py +++ b/api/resources/summarization_gene_expression.py @@ -233,14 +233,13 @@ def post(self): filename = secure_filename(file.filename) key = request.headers.get("X-Api-Key") overwrite = request.form.get("overwrite") - addToDb = request.form.get("addToDb") email = request.form.get("email") if overwrite is True: overwrite = "replace" else: overwrite = "append" # Create folder for user data if it doesn't exist - dirName = os.path.join(DATA_FOLDER, secure_filename(key)) + dirName = os.path.join("/DATA/users/www-data/", secure_filename(key)) if not os.path.exists(dirName): os.makedirs(dirName) file.save(os.path.join(dirName, secure_filename(filename))) @@ -258,9 +257,6 @@ def post(self): + """, "tsvUpload.overwrite": """ + overwrite - + """, - "tsvUpload.addToDb": """ - + addToDb + """, "tsvUpload.email": """ + email @@ -293,13 +289,12 @@ def post(self): filename = secure_filename(file.filename) key = request.headers.get("X-Api-Key") overwrite = request.form.get("overwrite") - addToDb = request.form.get("addToDb") email = request.form.get("email") if overwrite is True: overwrite = "replace" else: overwrite = "append" - dirName = os.path.join(DATA_FOLDER, secure_filename(key)) + dirName = os.path.join("/DATA/users/www-data/", secure_filename(key)) if not os.path.exists(dirName): os.makedirs(dirName) file.save(os.path.join(dirName, secure_filename(filename))) @@ -317,10 +312,7 @@ def post(self): "csvUpload.overwrite": """ + overwrite + """, - "csvUpload.addToDb": """ - + addToDb - + """, - "csvUpload.addToDb": """ + "csvUpload.email": """ + email + """ } From ddd89e9e532145f1415bb04bb5093ca46394fe6a Mon Sep 17 00:00:00 2001 From: asherpasha Date: Fri, 14 Jan 2022 10:33:20 -0500 Subject: [PATCH 32/33] Requirements update. --- .github/workflows/bar-api.yml | 2 +- Dockerfile | 2 +- api/resources/efp_image.py | 3 +-- docs/requirements.txt | 12 ++++++------ requirements.txt | 12 ++++++------ 5 files changed, 15 insertions(+), 16 deletions(-) diff --git a/.github/workflows/bar-api.yml b/.github/workflows/bar-api.yml index 5222361f..a2220262 100644 --- a/.github/workflows/bar-api.yml +++ b/.github/workflows/bar-api.yml @@ -13,7 +13,7 @@ jobs: runs-on: Ubuntu-20.04 strategy: matrix: - python-version: [3.7, 3.8, 3.9, 3.10.0] + python-version: [3.7, 3.8, 3.9, 3.10.1] services: redis: diff --git a/Dockerfile b/Dockerfile index 97167712..169eec22 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.10.0-bullseye +FROM python:3.10.1-bullseye WORKDIR /usr/src/app diff --git a/api/resources/efp_image.py b/api/resources/efp_image.py index f86e3286..01863b21 100644 --- a/api/resources/efp_image.py +++ b/api/resources/efp_image.py @@ -34,8 +34,7 @@ def get(self): @efp_image.route("////") @efp_image.route( - "/////", - doc=False, + "/////" ) class eFPImage(Resource): @efp_image.param("efp", _in="path", default="efp_arabidopsis") diff --git a/docs/requirements.txt b/docs/requirements.txt index eeb1665d..b37cd8a3 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -2,21 +2,21 @@ alabaster==0.7.12 Babel==2.9.1 beautifulsoup4==4.10.0 certifi==2021.10.8 -charset-normalizer==2.0.9 +charset-normalizer==2.0.10 docutils==0.17.1 -furo==2021.11.23 +furo==2022.1.2 idna==3.3 imagesize==1.3.0 Jinja2==3.0.3 MarkupSafe==2.0.1 packaging==21.3 -Pygments==2.10.0 +Pygments==2.11.2 pyparsing==3.0.6 pytz==2021.3 -requests==2.26.0 +requests==2.27.1 snowballstemmer==2.2.0 soupsieve==2.3.1 -Sphinx==4.3.1 +Sphinx==4.3.2 sphinx-copybutton==0.4.0 sphinxcontrib-applehelp==1.0.2 sphinxcontrib-devhelp==1.0.2 @@ -24,4 +24,4 @@ sphinxcontrib-htmlhelp==2.0.0 sphinxcontrib-jsmath==1.0.1 sphinxcontrib-qthelp==1.0.3 sphinxcontrib-serializinghtml==1.1.5 -urllib3==1.26.7 +urllib3==1.26.8 diff --git a/requirements.txt b/requirements.txt index f6e5910e..f46ebf88 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,7 @@ black==20.8b1 certifi==2021.10.8 cffi==1.15.0 chardet==4.0.0 -charset-normalizer==2.0.9 +charset-normalizer==2.0.10 click==8.0.3 coverage==6.2 cryptography==36.0.1 @@ -24,15 +24,15 @@ idna==3.3 iniconfig==1.1.1 itsdangerous==2.0.1 Jinja2==3.0.3 -jsonschema==4.3.2 -limits==2.1.0 +jsonschema==4.4.0 +limits==2.2.0 MarkupSafe==2.0.1 marshmallow==3.14.1 mccabe==0.6.1 more-itertools==8.12.0 mypy-extensions==0.4.3 mysqlclient==2.1.0 -numpy==1.21.5 +numpy==1.22.0 packaging==21.3 pandas==1.3.5 pathspec==0.9.0 @@ -48,14 +48,14 @@ python-dateutil==2.8.2 pytz==2021.3 redis==4.1.0 regex==2021.11.10 -requests==2.26.0 +requests==2.27.1 scour==0.38.2 six==1.16.0 SQLAlchemy==1.4.29 toml==0.10.2 typed-ast==1.5.1 typing_extensions==4.0.1 -urllib3==1.26.7 +urllib3==1.26.8 wcwidth==0.2.5 Werkzeug==2.0.2 wrapt==1.13.3 From ea098a4c191febd3d7ab7c6ed161771e49833c82 Mon Sep 17 00:00:00 2001 From: asherpasha Date: Fri, 14 Jan 2022 10:40:10 -0500 Subject: [PATCH 33/33] Fixed numpy --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f46ebf88..8d526ef6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -32,7 +32,7 @@ mccabe==0.6.1 more-itertools==8.12.0 mypy-extensions==0.4.3 mysqlclient==2.1.0 -numpy==1.22.0 +numpy==1.21.5 packaging==21.3 pandas==1.3.5 pathspec==0.9.0