diff --git a/.flake8 b/.flake8 index 44ce22a9..6206af19 100644 --- a/.flake8 +++ b/.flake8 @@ -11,4 +11,6 @@ exclude = venv, env, .venv, + ./venv-docs, + docs, .env diff --git a/.github/workflows/bar-api.yml b/.github/workflows/bar-api.yml index 04ae5bfc..a2220262 100644 --- a/.github/workflows/bar-api.yml +++ b/.github/workflows/bar-api.yml @@ -13,7 +13,7 @@ jobs: runs-on: Ubuntu-20.04 strategy: matrix: - python-version: [3.7, 3.8, 3.9, 3.10.0] + python-version: [3.7, 3.8, 3.9, 3.10.1] services: redis: @@ -58,5 +58,4 @@ jobs: - name: "Upload coverage to Codecov" uses: codecov/codecov-action@v2 with: - fail_ci_if_error: true - token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos + fail_ci_if_error: false diff --git a/.gitignore b/.gitignore index c5e6bab0..7e08bbba 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ __pycache__/ build/ develop-eggs/ dist/ +docs/build/ downloads/ eggs/ .eggs/ @@ -106,6 +107,7 @@ celerybeat.pid .venv env/ venv/ +venv-docs/ ENV/ env.bak/ venv.bak/ @@ -136,3 +138,6 @@ dmypy.json **/.DS_Store .vscode/ +output/* +!output +!output/placeholder.txt diff --git a/Dockerfile b/Dockerfile index 97167712..169eec22 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.10.0-bullseye +FROM python:3.10.1-bullseye WORKDIR /usr/src/app diff --git a/README.md b/README.md index 8c2d986d..2278f4ff 100644 --- a/README.md +++ b/README.md @@ -3,75 +3,6 @@ **master**: [![Build Status](https://github.com/BioAnalyticResource/BAR_API/workflows/BAR-API/badge.svg?branch=master)](https://github.com/BioAnalyticResource/BAR_API/actions?query=branch%3Amaster) [![codecov](https://codecov.io/gh/BioAnalyticResource/BAR_API/branch/master/graph/badge.svg?token=QSYUWTRYEV)](https://codecov.io/gh/BioAnalyticResource/BAR_API) **dev**: [![Build Status](https://github.com/BioAnalyticResource/BAR_API/workflows/BAR-API/badge.svg?branch=dev)](https://github.com/BioAnalyticResource/BAR_API/actions?query=branch%3Adev) [![codecov](https://codecov.io/gh/BioAnalyticResource/BAR_API/branch/dev/graph/badge.svg?token=QSYUWTRYEV)](https://codecov.io/gh/BioAnalyticResource/BAR_API) [![Website Status](https://img.shields.io/website?url=http%3A%2F%2Fbar.utoronto.ca%2Fapi%2F)](http://bar.utoronto.ca/api/) ![GitHub repo size](https://img.shields.io/github/repo-size/BioAnalyticResource/BAR_API) [![LGTM Alerts](https://img.shields.io/lgtm/alerts/github/BioAnalyticResource/BAR_API)](https://lgtm.com/projects/g/BioAnalyticResource/BAR_API/?mode=list) [![LGTM -Grade](https://img.shields.io/lgtm/grade/python/github/BioAnalyticResource/BAR_API)](https://lgtm.com/projects/g/BioAnalyticResource/BAR_API/latest/files/?sort=name&dir=ASC&mode=heatmap) [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) +Grade](https://img.shields.io/lgtm/grade/python/github/BioAnalyticResource/BAR_API)](https://lgtm.com/projects/g/BioAnalyticResource/BAR_API/latest/files/?sort=name&dir=ASC&mode=heatmap) [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) [![Documentation Status](https://readthedocs.org/projects/bar-api/badge/?version=latest)](https://bar-api.readthedocs.io/en/latest/?badge=latest) -This is the official repository for the Bio-Analytic Resource API. - -## Status - -Apart from Travis CI and testing on the live BAR, we frequently test the API on systems that are not BAR for cross-platform compatibility. The most recent test is on the following systems: - -* OpenBSD 6.7-CURRENT (Maria DB 10.4.12v1, Python 3.8.3 and Redis 6.0.5) -* FreeBSD 12.1-RELEASE-p6 (MySQL 8.0.20, Python 3.8.3 and Redis 5.0.9) - -## Run on your own computer with Docker - -1. Install [Docker](https://docs.docker.com/get-docker/) -2. Install [Docker Compose](https://docs.docker.com/compose/install/) -3. Install [Git](https://git-scm.com/downloads) -4. Clone this repository and change directory to ```BAR_API``` -5. Build docker images -``` -docker-compose build -``` -6. Run docker containers (-d is detached) -``` -docker-compose up -d -``` -7. Load ```http://localhost:5000/``` in a web browser. Enjoy :) - -## Run on your own computer without Docker - -1. Install [MySQL](https://www.mysql.com/products/community/) or [Maria DB](https://mariadb.com/downloads/). -2. Install [Redis](https://redis.io/download). -3. Install [Python](https://www.python.org/downloads/) or [Pypy](https://www.pypy.org/download.html). Note: Python 2 is not supported. -4. Install [Git](https://git-scm.com/downloads) -5. (Optional) On Debian based systems, you may also need to install ```libmysqlclient-dev``` and ```python3-dev```. On FreeBSD, you may need to install ```py38-sqlite3```. We will update this step as we come across more OS dependencies. -6. Clone this repository and change directory to ```BAR_API``` -7. Set up a virtual environment -``` -python3 -m venv venv -``` -8. Activate the virtual environment. Bash/Zsh: -``` -source venv/bin/activate -``` -csh/tcsh: -``` -source venv/bin/activate.csh -``` -9. Install requirements -``` -pip install --upgrade pip setuptools wheel -pip install -r requirements.txt -``` -10. Copy ```config/BAR_API.cgi``` to your preferred directory, for example: -``` -cp config/BAR_API.cgi ~/.config/ -``` -Add, update, and modify passwords and environment variables as needed. - -11. Copy ```./config/init.sh``` to BAR_API directory: -``` -cp config/init.sh . -``` -Change passwords in ```./init.sh``` and run this script to load the databases: -``` -./init.sh -``` -Then delete ```./init.sh``` - -12. Edit ```./api/__init__.py``` and update the location of your BAR_API.cfg file if you have changed it. -13. Run ```pytest```. Tests should pass if the system is set up correctly. -14. Run ```python app.py``` to start. -15. Load ```http://localhost:5000/``` in a web browser. Enjoy :) +This is the official repository for the Bio-Analytic Resource API. The API documentation can be found [here](https://bar-api.readthedocs.io/en/latest/). diff --git a/api/__init__.py b/api/__init__.py index 82e04936..ea91f1d7 100644 --- a/api/__init__.py +++ b/api/__init__.py @@ -20,14 +20,28 @@ def create_app(): print("We are now loading configuration.") bar_app.config.from_pyfile(os.getcwd() + "/config/BAR_API.cfg", silent=True) if bar_app.config.get("ADMIN_ENCRYPT_KEY"): - os.environ["ADMIN_ENCRYPT_KEY"] = bar_app.config.get("ADMIN_ENCRYPT_KEY") + os.environ["ADMIN_ENCRYPT_KEY"] = bar_app.config.get( + "TEST_ADMIN_ENCRYPT_KEY" + ) if bar_app.config.get("ADMIN_PASSWORD_FILE"): os.environ["ADMIN_PASSWORD_FILE"] = bar_app.config.get( - "ADMIN_PASSWORD_FILE" + "TEST_ADMIN_PASSWORD_FILE" ) + if bar_app.config.get("ADMIN_EMAIL"): + os.environ["ADMIN_EMAIL"] = bar_app.config.get("ADMIN_EMAIL") + if bar_app.config.get("EMAIL_PASS_KEY"): + os.environ["EMAIL_PASS_KEY"] = bar_app.config.get("EMAIL_PASS_KEY") + if bar_app.config.get("EMAIL_PASS_FILE"): + os.environ["EMAIL_PASS_FILE"] = bar_app.config.get("EMAIL_PASS_FILE") elif os.environ.get("BAR"): # The BAR bar_app.config.from_pyfile(os.environ.get("BAR_API_PATH"), silent=True) + if bar_app.config.get("ADMIN_EMAIL"): + os.environ["ADMIN_EMAIL"] = bar_app.config.get("ADMIN_EMAIL") + if bar_app.config.get("EMAIL_PASS_KEY"): + os.environ["EMAIL_PASS_KEY"] = bar_app.config.get("EMAIL_PASS_KEY") + if bar_app.config.get("EMAIL_PASS_FILE"): + os.environ["EMAIL_PASS_FILE"] = bar_app.config.get("EMAIL_PASS_FILE") if bar_app.config.get("ADMIN_ENCRYPT_KEY"): os.environ["ADMIN_ENCRYPT_KEY"] = bar_app.config.get("ADMIN_ENCRYPT_KEY") if bar_app.config.get("ADMIN_PASSWORD_FILE"): @@ -51,6 +65,12 @@ def create_app(): os.environ["ADMIN_PASSWORD_FILE"] = bar_app.config.get( "ADMIN_PASSWORD_FILE" ) + if bar_app.config.get("ADMIN_EMAIL"): + os.environ["ADMIN_EMAIL"] = bar_app.config.get("ADMIN_EMAIL") + if bar_app.config.get("EMAIL_PASS_KEY"): + os.environ["EMAIL_PASS_KEY"] = bar_app.config.get("EMAIL_PASS_KEY") + if bar_app.config.get("EMAIL_PASS_FILE"): + os.environ["EMAIL_PASS_FILE"] = bar_app.config.get("EMAIL_PASS_FILE") if bar_app.config.get("DRIVE_LIST_KEY"): os.environ["DRIVE_LIST_KEY"] = bar_app.config.get("DRIVE_LIST_KEY") if bar_app.config.get("DRIVE_LIST_FILE"): @@ -103,6 +123,7 @@ def create_app(): from api.resources.gene_annotation import gene_annotation from api.resources.interactions import itrns from api.resources.gene_localizations import loc + from api.resources.efp_image import efp_image bar_api.add_namespace(gene_information) bar_api.add_namespace(rnaseq_gene_expression) @@ -115,6 +136,7 @@ def create_app(): bar_api.add_namespace(gene_annotation) bar_api.add_namespace(itrns) bar_api.add_namespace(loc) + bar_api.add_namespace(efp_image) bar_api.init_app(bar_app) return bar_app diff --git a/api/resources/api_manager.py b/api/resources/api_manager.py index 3073389b..0a7327ca 100644 --- a/api/resources/api_manager.py +++ b/api/resources/api_manager.py @@ -5,11 +5,16 @@ from flask_restx import Namespace, Resource from datetime import datetime from sqlalchemy.exc import SQLAlchemyError +from sqlalchemy.types import String, Float import os import uuid import requests import pandas from cryptography.fernet import Fernet +from smtplib import SMTP_SSL +from ssl import create_default_context +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText CAPTCHA_KEY_FILE = "/home/bpereira/data/bar.summarization/key" @@ -32,6 +37,56 @@ def check_admin_pass(password): else: return False + @staticmethod + def validate_captcha(value): + """Validates a reCaptcha value using our secret token""" + if os.environ.get("BAR"): + with open(CAPTCHA_KEY_FILE, "rb") as f: + for line in f: + key = line + if key: + ret = requests.post( + "https://www.google.com/recaptcha/api/siteverify", + data={"secret": key, "response": value}, + ) + return ret.json()["success"] + else: + return False + else: + return True + + @staticmethod + def send_email_notification(): + if os.environ.get("BAR"): + with open(os.environ.get("ADMIN_EMAIL"), "r") as f: + for line in f: + recipient = line + port = 465 + key = os.environ.get("EMAIL_PASS_KEY") + cipher_suite = Fernet(key) + with open(os.environ.get("EMAIL_PASS_FILE"), "rb") as f: + for line in f: + encrypted_key = line + uncipher_text = cipher_suite.decrypt(encrypted_key) + password = bytes(uncipher_text).decode("utf-8") + context = create_default_context() + smtp_server = "smtp.gmail.com" + sender_email = "bar.summarization@gmail.com" + subject = "[Bio-Analytic Resource] New API key request" + text = """\ + There is a new API key request. + You can approve or reject it at http://bar.utoronto.ca/~bpereira/webservices/bar-request-manager/build/index.html + """ + m_text = MIMEText(text, _subtype="plain", _charset="UTF-8") + msg = MIMEMultipart() + msg["From"] = sender_email + msg["To"] = recipient + msg["Subject"] = subject + msg.attach(m_text) + with SMTP_SSL(smtp_server, port, context=context) as server: + server.login("bar.summarization@gmail.com", password) + server.sendmail(sender_email, recipient, msg.as_string()) + @api_manager.route("/validate_admin_password", methods=["POST"], doc=False) class ApiManagerValidate(Resource): @@ -72,22 +127,27 @@ def post(self): class ApiManagerRequest(Resource): def post(self): if request.method == "POST": - response_json = request.get_json() - df = pandas.DataFrame.from_records([response_json]) - con = db.get_engine(bind="summarization") - try: - reqs = Requests() - users = Users() - row_req = reqs.query.filter_by(email=df.email[0]).first() - row_users = users.query.filter_by(email=df.email[0]).first() - - if row_req is None and row_users is None: - df.to_sql("requests", con, if_exists="append", index=False) - return BARUtils.success_exit("Data added") - else: - return BARUtils.error_exit("E-mail already in use"), 409 - except SQLAlchemyError: - return BARUtils.error_exit("Internal server error"), 500 + captchaVal = request.headers.get("captchaVal") + if ApiManagerUtils.validate_captcha(captchaVal): + response_json = request.get_json() + df = pandas.DataFrame.from_records([response_json]) + con = db.get_engine(bind="summarization") + try: + reqs = Requests() + users = Users() + row_req = reqs.query.filter_by(email=df.email[0]).first() + row_users = users.query.filter_by(email=df.email[0]).first() + + if row_req is None and row_users is None: + df.to_sql("requests", con, if_exists="append", index=False) + ApiManagerUtils.send_email_notification() + return BARUtils.success_exit("Data added") + else: + return BARUtils.error_exit("E-mail already in use"), 409 + except SQLAlchemyError: + return BARUtils.error_exit("Internal server error"), 500 + else: + return BARUtils.error_exit("Failed Captcha verification") @api_manager.route("/get_pending_requests", methods=["POST"], doc=False) @@ -167,37 +227,34 @@ def post(self): "date_added": datetime.now(), "status": "user", "api_key": key, - "uses_left": 25, + "uses_left": 100, } ) for row in rows ] df = pandas.DataFrame.from_records([values[0]]) + values_df = pandas.DataFrame(columns=["Gene", "Sample", "Value"]) con = db.get_engine(bind="summarization") try: df.to_sql("users", con, if_exists="append", index=False) + values_df.to_sql( + key, + con, + index_label="index", + dtype={ + values_df.index.name: String(42), + "Gene": String(32), + "Sample": String(32), + "Value": Float, + }, + if_exists="append", + index=True, + ) el = table.query.filter_by(email=email).one() db.session.delete(el) + db.session.commit() except SQLAlchemyError: return BARUtils.error_exit("Internal server error"), 500 return BARUtils.success_exit(key) else: return BARUtils.error_exit("Forbidden"), 403 - - -@api_manager.route("/validate_captcha", methods=["POST"], doc=False) -class ApiManagerCaptchaValidate(Resource): - def post(self): - """Validates a reCaptcha value using our secret token""" - if request.method == "POST": - json = request.get_json() - value = json["response"] - key = os.environ.get("CAPTCHA_KEY_FILE") - if key: - ret = requests.post( - "https://www.google.com/recaptcha/api/siteverify", - data={"secret": key, "response": value}, - ) - return BARUtils.success_exit(ret.text) - else: - return BARUtils.error_exit("Forbidden: CAPTCHA key is not found"), 403 diff --git a/api/resources/efp_image.py b/api/resources/efp_image.py new file mode 100644 index 00000000..01863b21 --- /dev/null +++ b/api/resources/efp_image.py @@ -0,0 +1,140 @@ +import base64 +import re +import requests +import random +import os +import time +import redis.exceptions +from flask_restx import Namespace, Resource +from markupsafe import escape +from flask import send_from_directory +from api.utils.bar_utils import BARUtils +from api.utils.efp_utils import eFPUtils + +efp_image = Namespace( + "eFP Image", description="eFP Image generation service", path="/efp_image" +) + + +@efp_image.route("/") +class eFPImageList(Resource): + def get(self): + """This end point returns the list of species available""" + # This are the only species available so far + # If this is updated, update efp_utils.py and unit tests as well + species = [ + "efp_arabidopsis", + "efp_cannabis", + "efp_arachis", + "efp_soybean", + "efp_maize", + ] + return BARUtils.success_exit(species) + + +@efp_image.route("////") +@efp_image.route( + "/////" +) +class eFPImage(Resource): + @efp_image.param("efp", _in="path", default="efp_arabidopsis") + @efp_image.param("view", _in="path", default="Developmental_Map") + @efp_image.param("mode", _in="path", default="Absolute") + @efp_image.param("gene_1", _in="path", default="At1g01010") + @efp_image.param("gene_2", _in="path", default="At3g27340") + def get(self, efp="", view="", mode="", gene_1="", gene_2=""): + """This end point returns eFP images.""" + # Escape input data + efp = escape(efp) + view = escape(view) + mode = escape(mode) + gene_1 = escape(gene_1) + gene_2 = escape(gene_2) + + validation = eFPUtils.is_efp_input_valid(efp, view, mode, gene_1, gene_2) + if validation[0] is False: + return BARUtils.error_exit(validation[1]), 400 + + # Data are valid. Clear directory before running + for file in os.listdir("output"): + # Full file name is required at this point + file = os.path.join("output", file) + + # Check if it is a png file and is greater than 5 minutes old + if ( + os.path.isfile(file) + and (os.path.splitext(file)[1] == ".png") + and (os.stat(file).st_mtime < (time.time() - 5 * 60)) + ): + os.remove(file) + + # Check if request is cached + try: + r = BARUtils.connect_redis() + key = "BAR_API_efp_image_" + "_".join([efp, view, mode, gene_1, gene_2]) + efp_image_base64 = r.get(key) + except redis.exceptions.ConnectionError: + # Failed redis connection + r = None + key = None + efp_image_base64 = None + + if efp_image_base64 is None: + # Request is not cached + # Run eFP. Note, this is currently running from home directory! + efp_url = ( + "https://bar.utoronto.ca/~asher/python3/" + + efp + + "/cgi-bin/efpWeb.cgi?dataSource=" + + view + + "&mode=" + + mode + + "&primaryGene=" + + gene_1 + + "&secondaryGene=" + + gene_2 + + "&grey_low=None&grey_stddev=None" + ) + efp_html = requests.get(efp_url) + + # Now search for something like NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.https://www.sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 00000000..b37cd8a3 --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,27 @@ +alabaster==0.7.12 +Babel==2.9.1 +beautifulsoup4==4.10.0 +certifi==2021.10.8 +charset-normalizer==2.0.10 +docutils==0.17.1 +furo==2022.1.2 +idna==3.3 +imagesize==1.3.0 +Jinja2==3.0.3 +MarkupSafe==2.0.1 +packaging==21.3 +Pygments==2.11.2 +pyparsing==3.0.6 +pytz==2021.3 +requests==2.27.1 +snowballstemmer==2.2.0 +soupsieve==2.3.1 +Sphinx==4.3.2 +sphinx-copybutton==0.4.0 +sphinxcontrib-applehelp==1.0.2 +sphinxcontrib-devhelp==1.0.2 +sphinxcontrib-htmlhelp==2.0.0 +sphinxcontrib-jsmath==1.0.1 +sphinxcontrib-qthelp==1.0.3 +sphinxcontrib-serializinghtml==1.1.5 +urllib3==1.26.8 diff --git a/docs/source/conf.py b/docs/source/conf.py new file mode 100644 index 00000000..9121f1c3 --- /dev/null +++ b/docs/source/conf.py @@ -0,0 +1,54 @@ +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +# import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) + + +# -- Project information ----------------------------------------------------- + +project = "BAR API" +copyright = "2021, BAR Developers" +author = "BAR Developers" + +# The full version, including alpha/beta/rc tags +release = "0.0.1" + + +# -- General configuration --------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = ["sphinx_copybutton"] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ["_templates"] + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = [] + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = "furo" + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ["_static"] diff --git a/docs/source/developer_guide.rst b/docs/source/developer_guide.rst new file mode 100644 index 00000000..c887f61b --- /dev/null +++ b/docs/source/developer_guide.rst @@ -0,0 +1,57 @@ +Developer Guide +=============== + +Basic Requirements +------------------ +Developers need to learn Python 3 programming language (tutorial: `Python3`_ ) and GitHub. + +Core Dependencies +----------------- + +**Flask**: The BAR API uses Python Flask (tutorial: `Flask`_). We use a class based approach to organize code. + +**Flask-restx**: Flask RestX is used to create endpoints and generate swagger fontend (tutorial: `Flask-restx`_ ). + +**Flask-Caching**: This module is used to cache queries (tutorial: `Flask-Caching`_). We used Redis as a key-value store. + +**Flask-Limiter**: This module is used to rate-limit the usage of API end points. This will prevent server from overloading (tutorial: `Flask-Limiter`_ ) + +**Marshmallow**: This module is only used to validate JSON schema for POST requests (tutorial: `Marshmallow`_). + +**MarkupSafe**: This is used to validate GET request inputs (tutorial: `MarkupSafe`_). + +**Flask-SQLAlchemy**: This is used to provide database connectivity and queries (tutorial: `Flask-SQLAlchemy`_). This is based on Python SQLAlchemy (tutorial: `SQLAlchemy`_) + +**Flake8**: This is used to see if there are no style errors in code. Just run ``flake8`` command. + +**Pytest**: This module (tutorial: `Pytest`_) along with Flask testing framework (tutorial: `Flask-Testing`_) is used to unit test API endpoints. + +**Coverage**: This module show the test code coverage, that is what parts of code are tested by unit tests (tutorial: `Coverage`_) + +**Black**: This module is used to format and clean up code. Just run ``black .`` command. + +Online CI/CD Pipeline +--------------------- + +**GitHub Actions**: This runs automated testing on GitHub (tutorial: `GitHub-Actions`_). + +**LGTM**: This is for code quality testing. + +**Codecov**: This report code coverage. + +**Read the Docs**: This hosts documentation (tutorial: `readthedocs`_). + +.. _Python3: https://docs.python.org/3/tutorial/index.html +.. _Flask: https://flask.palletsprojects.com/en/2.0.x/quickstart/ +.. _Flask-Caching: https://flask-caching.readthedocs.io/en/latest/index.html +.. _Flask-Limiter: https://flask-limiter.readthedocs.io/en/master/#using-flask-pluggable-views +.. _Flask-restx: https://flask-restx.readthedocs.io/en/latest/ +.. _Marshmallow: https://marshmallow.readthedocs.io/en/stable/quickstart.html +.. _MarkupSafe: https://pypi.org/project/MarkupSafe/ +.. _Flask-SQLAlchemy: https://flask-sqlalchemy.palletsprojects.com/en/2.x/ +.. _SQLAlchemy: https://docs.sqlalchemy.org/en/14/ +.. _Pytest: https://docs.pytest.org/en/latest/getting-started.html +.. _Flask-Testing: https://flask.palletsprojects.com/en/2.0.x/testing/ +.. _Coverage: https://coverage.readthedocs.io/en/6.2/ +.. _GitHub-Actions: https://docs.github.com/en/actions/quickstart +.. _readthedocs: https://docs.readthedocs.io/en/stable/tutorial/ diff --git a/docs/source/index.rst b/docs/source/index.rst new file mode 100644 index 00000000..7d8cc810 --- /dev/null +++ b/docs/source/index.rst @@ -0,0 +1,16 @@ +.. BAR API documentation master file, created by + sphinx-quickstart on Fri Nov 26 19:30:59 2021. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to BAR API's documentation! +=================================== + +This is the documentation for the BAR API. + +.. toctree:: + :maxdepth: 2 + + installation_guide + user_guide + developer_guide \ No newline at end of file diff --git a/docs/source/installation_guide.rst b/docs/source/installation_guide.rst new file mode 100644 index 00000000..e3b78a1e --- /dev/null +++ b/docs/source/installation_guide.rst @@ -0,0 +1,95 @@ +Installation Guide +================== + +Run on your own computer with Docker +------------------------------------ +1. Install `Docker`_ +2. Install `Docker Compose`_ +3. Install `Git`_ +4. Clone this repository and change directory to ``BAR_API`` +5. Build docker images + +.. code-block:: bash + + docker-compose build + +6. Run docker containers (-d is detached) + +.. code-block:: bash + + docker-compose up -d + +7. Load ``http://localhost:5000/`` in a web browser. Enjoy :) + +Run on your own computer without Docker +--------------------------------------- + +1. Install `MySQL`_ or `Maria DB`_ +2. Install `Redis`_ +3. Install `Python`_ or `Pypy`_. Note: Python 2 is not supported. +4. Install `Git`_ +5. (Optional) On Debian based systems, you may also need to install ``libmysqlclient-dev`` and ``python3-dev``. On FreeBSD, you may need to install ``py38-sqlite3``. We will update this step as we come across more OS dependencies. +6. Clone this repository and change directory to ``BAR_API`` +7. Set up a virtual environment + +.. code-block:: bash + + python3 -m venv venv + +8. Activate the virtual environment. Bash/Zsh: + +.. code-block:: bash + + source venv/bin/activate + +csh/tcsh: + +.. code-block:: bash + + source venv/bin/activate.csh + +9. Install requirements + +.. code-block:: bash + + pip install --upgrade pip setuptools wheel + pip install -r requirements.txt + +10. Copy ``config/BAR_API.cgi`` to your preferred directory, for example: + +.. code-block:: bash + + cp config/BAR_API.cgi ~/.config/ + +Add, update, and modify passwords and environment variables as needed. + +11. Copy ``./config/init.sh`` to BAR_API directory: + +.. code-block:: bash + + cp config/init.sh . + +Change passwords in ``./init.sh`` and run this script to load the databases: + +.. code-block:: bash + + ./init.sh + +Then delete ``./init.sh``. + +12. Edit ``./api/__init__.py`` and update the location of your BAR_API.cfg file if you have changed it. + +13. Run ``pytest``. Tests should pass if the system is set up correctly. + +14. Run ``python app.py`` to start. + +15. Load ``http://localhost:5000/`` in a web browser. Enjoy :) + +.. _Docker: https://docs.docker.com/get-docker/ +.. _Docker Compose: https://docs.docker.com/compose/install/ +.. _Git: https://git-scm.com/downloads +.. _MySQL: https://www.mysql.com/products/community/ +.. _Maria DB: https://mariadb.com/downloads/ +.. _Redis: https://redis.io/download +.. _Python: https://www.python.org/downloads/ +.. _Pypy: https://www.pypy.org/download.html diff --git a/docs/source/user_guide.rst b/docs/source/user_guide.rst new file mode 100644 index 00000000..c4570fd4 --- /dev/null +++ b/docs/source/user_guide.rst @@ -0,0 +1,4 @@ +User Guide +========== + +Coming soon ... \ No newline at end of file diff --git a/output/placeholder.txt b/output/placeholder.txt new file mode 100644 index 00000000..e118c242 --- /dev/null +++ b/output/placeholder.txt @@ -0,0 +1 @@ +This file exists simply so the output folder will be created when cloning repository. diff --git a/requirements.txt b/requirements.txt index 7c26d032..8d526ef6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,21 +1,21 @@ aniso8601==9.0.1 appdirs==1.4.4 -attrs==21.2.0 +attrs==21.4.0 black==20.8b1 certifi==2021.10.8 cffi==1.15.0 chardet==4.0.0 -charset-normalizer==2.0.8 +charset-normalizer==2.0.10 click==8.0.3 -coverage==6.1.2 -cryptography==36.0.0 +coverage==6.2 +cryptography==36.0.1 Deprecated==1.2.13 docopt==0.6.2 flake8==4.0.1 Flask==2.0.2 Flask-Caching==1.10.1 Flask-Cors==3.0.10 -Flask-Limiter==1.4 +Flask-Limiter==2.0.4 flask-marshmallow==0.14.0 flask-restx==0.5.1 Flask-SQLAlchemy==2.5.1 @@ -24,17 +24,17 @@ idna==3.3 iniconfig==1.1.1 itsdangerous==2.0.1 Jinja2==3.0.3 -jsonschema==4.2.1 -limits==1.5.1 +jsonschema==4.4.0 +limits==2.2.0 MarkupSafe==2.0.1 marshmallow==3.14.1 mccabe==0.6.1 more-itertools==8.12.0 mypy-extensions==0.4.3 mysqlclient==2.1.0 -numpy==1.21.4 +numpy==1.21.5 packaging==21.3 -pandas==1.3.4 +pandas==1.3.5 pathspec==0.9.0 pluggy==1.0.0 py==1.11.0 @@ -46,16 +46,16 @@ pyrsistent==0.18.0 pytest==6.2.5 python-dateutil==2.8.2 pytz==2021.3 -redis==4.0.2 +redis==4.1.0 regex==2021.11.10 -requests==2.26.0 +requests==2.27.1 scour==0.38.2 six==1.16.0 -SQLAlchemy==1.4.27 +SQLAlchemy==1.4.29 toml==0.10.2 -typed-ast==1.5.0 -typing_extensions==4.0.0 -urllib3==1.26.7 +typed-ast==1.5.1 +typing_extensions==4.0.1 +urllib3==1.26.8 wcwidth==0.2.5 Werkzeug==2.0.2 wrapt==1.13.3 diff --git a/tests/resources/test_efp_image.py b/tests/resources/test_efp_image.py new file mode 100644 index 00000000..0b77e550 --- /dev/null +++ b/tests/resources/test_efp_image.py @@ -0,0 +1,86 @@ +from api import app +from unittest import TestCase + + +class TestIntegrations(TestCase): + def setUp(self): + self.app_client = app.test_client() + + def test_get_efp_image_list(self): + """This function tests the gene alias list get function + :return: + """ + response = self.app_client.get("/efp_image/") + expected = { + "wasSuccessful": True, + "data": [ + "efp_arabidopsis", + "efp_cannabis", + "efp_arachis", + "efp_soybean", + "efp_maize", + ], + } + self.assertEqual(response.json, expected) + + def test_get_efp_image(self): + """This function test eFP image endpoint get request + :return: + """ + # Test absolute modes in the beginning + # A very basic test for Arabidopsis requests + # https://bar.utoronto.ca/api/efp_image/efp_cannabis/Cannabis_Atlas/Absolute/AGQN03000001 + response = self.app_client.get( + "/efp_image/efp_cannabis/Cannabis_Atlas/Absolute/AGQN03000001" + ) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.content_type, "image/png") + + # Now rerun for Cached requests. An image should be return from the cache + response = self.app_client.get( + "/efp_image/efp_cannabis/Cannabis_Atlas/Absolute/AGQN03000001" + ) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.content_type, "image/png") + + # Test for invalid species: + response = self.app_client.get( + "/efp_image/abc/Developmental_Map/Absolute/At1g01010" + ) + expected = {"wasSuccessful": False, "error": "Invalid eFP."} + self.assertEqual(response.json, expected) + + # Test for eFP view name + response = self.app_client.get( + "/efp_image/efp_arabidopsis/ab.!c/Absolute/At1g01010" + ) + expected = {"wasSuccessful": False, "error": "Invalid eFP View name."} + self.assertEqual(response.json, expected) + + # Test for eFP mode + response = self.app_client.get("/efp_image/efp_arabidopsis/Root/abc/At1g01010") + expected = {"wasSuccessful": False, "error": "Invalid eFP mode."} + self.assertEqual(response.json, expected) + + # Test for gene 1 using Arabidopsis + response = self.app_client.get( + "/efp_image/efp_arabidopsis/Root/Absolute/At1g0101X" + ) + expected = {"wasSuccessful": False, "error": "Gene 1 is invalid."} + self.assertEqual(response.json, expected) + + response = self.app_client.get( + "/efp_image/efp_arabidopsis/Developmental_Map/Absolute/At1g01011" + ) + expected = { + "wasSuccessful": False, + "error": "Failed to retrieve image. Data for the given gene may not exist.", + } + self.assertEqual(response.json, expected) + + # Test for gene 1 using Arabidopsis + response = self.app_client.get( + "/efp_image/efp_arabidopsis/Root/Compare/At1g01010/Abc" + ) + expected = {"wasSuccessful": False, "error": "Gene 2 is invalid."} + self.assertEqual(response.json, expected) diff --git a/tests/resources/test_interactions.py b/tests/resources/test_interactions.py new file mode 100644 index 00000000..0efde6b5 --- /dev/null +++ b/tests/resources/test_interactions.py @@ -0,0 +1,58 @@ +from api import app +from unittest import TestCase + + +class TestIntegrations(TestCase): + + maxDiff = None + + def setUp(self): + self.app_client = app.test_client() + + def test_get_itrns(self): + """ + This function test retrieving protein interactions for various species' genes. + """ + + # Valid request rice + response = self.app_client.get("/interactions/rice/LOC_Os01g52560") + expected = { + "wasSuccessful": True, + "data": [ + { + "protein_1": "LOC_Os01g01080", + "protein_2": "LOC_Os01g52560", + "total_hits": 1, + "Num_species": 1, + "Quality": 1, + "pcc": 0.65, + }, + { + "protein_1": "LOC_Os01g52560", + "protein_2": "LOC_Os01g73310", + "total_hits": 1, + "Num_species": 1, + "Quality": 1, + "pcc": -0.116, + }, + ], + } + self.assertEqual(response.json, expected) + + # Invalid species + response = self.app_client.get("/interactions/poplar/abc") + expected = {"wasSuccessful": False, "error": "Invalid species or gene ID"} + self.assertEqual(response.json, expected) + + # Invalid gene id + response = self.app_client.get("/interactions/rice/abc") + expected = {"wasSuccessful": False, "error": "Invalid species or gene ID"} + self.assertEqual(response.json, expected) + + # Gene does not exist + response = self.app_client.get("/interactions/rice/LOC_Os01g52565") + expected = { + "wasSuccessful": False, + "error": "There are no data found for the given gene", + } + self.assertEqual(response.json, expected) diff --git a/tests/resources/test_localizations.py b/tests/resources/test_localizations.py new file mode 100644 index 00000000..a7f163ed --- /dev/null +++ b/tests/resources/test_localizations.py @@ -0,0 +1,94 @@ +from api import app +from unittest import TestCase +import json + + +class TestIntegrations(TestCase): + def setUp(self): + self.app_client = app.test_client() + + def test_get_loc(self): + """ + This function test retrieving subcellular localizations for various species' genes via GET. + """ + + # Valid request rice + response = self.app_client.get("/loc/rice/LOC_Os01g52560.1") + expected = { + "wasSuccessful": True, + "data": { + "gene": "LOC_Os01g52560.1", + "predicted_location": "Cellmembrane,Chloroplast", + }, + } + self.assertEqual(response.json, expected) + + # Invalid species + response = self.app_client.get("/loc/poplar/LOC_Os01g52560.1") + expected = {"wasSuccessful": False, "error": "Invalid species or gene ID"} + self.assertEqual(response.json, expected) + + # Invalid gene id + response = self.app_client.get("/loc/rice/abc") + expected = {"wasSuccessful": False, "error": "Invalid species or gene ID"} + self.assertEqual(response.json, expected) + + # Gene does not exist + response = self.app_client.get("/loc/rice/LOC_Os01g52561.1") + expected = { + "wasSuccessful": False, + "error": "There are no data found for the given gene", + } + self.assertEqual(response.json, expected) + + def test_post_loc(self): + """ + This function test retrieving subcellular localizations for various species' genes via POST. + """ + + # Valid request + response = self.app_client.post( + "/loc/", + json={"species": "rice", "genes": ["LOC_Os01g01080.1", "LOC_Os01g52560.1"]}, + ) + data = json.loads(response.get_data(as_text=True)) + expected = { + "wasSuccessful": True, + "data": { + "LOC_Os01g01080.1": ["Endoplasmic reticulum"], + "LOC_Os01g52560.1": ["Cellmembrane,Chloroplast"], + }, + } + self.assertEqual(data, expected) + + # Invalid species + response = self.app_client.post( + "/loc/", + json={ + "species": "poplar", + "genes": ["LOC_Os01g01080.1", "LOC_Os01g52560.1"], + }, + ) + data = json.loads(response.get_data(as_text=True)) + expected = {"wasSuccessful": False, "error": "Invalid species"} + self.assertEqual(data, expected) + + # Invalid gene ID + response = self.app_client.post( + "/loc/", json={"species": "rice", "genes": ["abc", "xyz"]} + ) + data = json.loads(response.get_data(as_text=True)) + expected = {"wasSuccessful": False, "error": "Invalid gene id"} + self.assertEqual(data, expected) + + # No data for valid gene IDs + response = self.app_client.post( + "/loc/", + json={"species": "rice", "genes": ["LOC_Os01g01085.1", "LOC_Os01g52565.1"]}, + ) + data = json.loads(response.get_data(as_text=True)) + expected = { + "wasSuccessful": False, + "error": "No data for the given species/genes", + } + self.assertEqual(data, expected) diff --git a/tests/utils/test_bar_utils.py b/tests/utils/test_bar_utils.py index ac9ab76b..f7d06e64 100644 --- a/tests/utils/test_bar_utils.py +++ b/tests/utils/test_bar_utils.py @@ -28,6 +28,11 @@ def test_is_arabidopsis_gene_valid(self): result = BARUtils.is_arabidopsis_gene_valid("At1g01010.11") self.assertFalse(result) + def test_is_tomato_gene_valid(self): + # For some reason, coverage is saying that we need this test + result = BARUtils.is_tomato_gene_valid("Solyc04g014530") + self.assertTrue(result) + def test_is_integer(self): # Valid result result = BARUtils.is_integer("5") diff --git a/tests/utils/test_efp_utils.py b/tests/utils/test_efp_utils.py new file mode 100644 index 00000000..d3bc1a7a --- /dev/null +++ b/tests/utils/test_efp_utils.py @@ -0,0 +1,169 @@ +from unittest import TestCase +from api.utils.efp_utils import eFPUtils + + +class UtilsUnitTest(TestCase): + def test_is_efp_input_valid(self): + """Tests for eFP input data""" + + # eFP Arabidopsis compare mode + result = eFPUtils.is_efp_input_valid( + "efp_arabidopsis", "Root", "Compare", "At1g01010", "At1g01030" + ) + self.assertTrue(result[0]) + self.assertIsNone(result[1]) + + result = eFPUtils.is_efp_input_valid( + "efp_arabidopsis", "Root", "Compare", "At1g01010", "Abc" + ) + expected = "Gene 2 is invalid." + self.assertFalse(result[0]) + self.assertEqual(result[1], expected) + + # eFP Cannabis gene1 + result = eFPUtils.is_efp_input_valid( + "efp_cannabis", "Cannabis_Atlas", "Absolute", "AGQN03000001" + ) + self.assertTrue(result[0]) + self.assertIsNone(result[1]) + + result = eFPUtils.is_efp_input_valid( + "efp_cannabis", "Cannabis_Atlas", "Absolute", "Abc" + ) + expected = "Gene 1 is invalid." + self.assertFalse(result[0]) + self.assertEqual(result[1], expected) + + # eFP Cannabis gene2 + result = eFPUtils.is_efp_input_valid( + "efp_cannabis", "Cannabis_Atlas", "Compare", "AGQN03000001", "AGQN03000012" + ) + self.assertTrue(result[0]) + self.assertIsNone(result[1]) + + result = eFPUtils.is_efp_input_valid( + "efp_cannabis", "Cannabis_Atlas", "Compare", "AGQN03000001", "Abc" + ) + expected = "Gene 2 is invalid." + self.assertFalse(result[0]) + self.assertEqual(result[1], expected) + + # eFP Arachis Absolute + result = eFPUtils.is_efp_input_valid( + "efp_arachis", "Arachis_Atlas", "Absolute", "Adur10002_comp0_c0_seq1" + ) + self.assertTrue(result[0]) + self.assertIsNone(result[1]) + + result = eFPUtils.is_efp_input_valid( + "efp_arachis", "Arachis_Atlas", "Absolute", "Abc" + ) + expected = "Gene 1 is invalid." + self.assertFalse(result[0]) + self.assertEqual(result[1], expected) + + # eFP Arachis gene 2 + result = eFPUtils.is_efp_input_valid( + "efp_arachis", + "Arachis_Atlas", + "Compare", + "Adur10002_comp0_c0_seq1", + "Adur10002_comp0_c0_seq11", + ) + self.assertTrue(result[0]) + self.assertIsNone(result[1]) + + result = eFPUtils.is_efp_input_valid( + "efp_arachis", "Arachis_Atlas", "Compare", "Adur10002_comp0_c0_seq1", "Abc" + ) + expected = "Gene 2 is invalid." + self.assertFalse(result[0]) + self.assertEqual(result[1], expected) + + # eFP Soybean Absolute + result = eFPUtils.is_efp_input_valid( + "efp_soybean", "soybean", "Absolute", "Glyma06g47400" + ) + self.assertTrue(result[0]) + self.assertIsNone(result[1]) + + result = eFPUtils.is_efp_input_valid( + "efp_soybean", "soybean", "Absolute", "Abc" + ) + expected = "Gene 1 is invalid." + self.assertFalse(result[0]) + self.assertEqual(result[1], expected) + + # eFP Soybean gene 2 + result = eFPUtils.is_efp_input_valid( + "efp_soybean", + "soybean", + "Compare", + "Glyma06g47400", + "Glyma06g47390", + ) + self.assertTrue(result[0]) + self.assertIsNone(result[1]) + + result = eFPUtils.is_efp_input_valid( + "efp_soybean", "soybean", "Compare", "Glyma06g47400", "Abc" + ) + expected = "Gene 2 is invalid." + self.assertFalse(result[0]) + self.assertEqual(result[1], expected) + + # Test if both gene are the same in eFP Comare mode + result = eFPUtils.is_efp_input_valid( + "efp_arachis", + "Arachis_Atlas", + "Compare", + "Adur10002_comp0_c0_seq1", + "Adur10002_comp0_c0_seq1", + ) + expected = "In compare mode, both genes should be different." + self.assertFalse(result[0]) + self.assertEqual(result[1], expected) + + # eFP Maize Absolute + result = eFPUtils.is_efp_input_valid( + "efp_maize", "maize_iplant", "Absolute", "Zm00001d046170" + ) + self.assertTrue(result[0]) + self.assertIsNone(result[1]) + + result = eFPUtils.is_efp_input_valid( + "efp_maize", "maize_iplant", "Absolute", "Abc" + ) + expected = "Gene 1 is invalid." + self.assertFalse(result[0]) + self.assertEqual(result[1], expected) + + # eFP Maize gene 2 + result = eFPUtils.is_efp_input_valid( + "efp_maize", + "maize_iplant", + "Compare", + "Zm00001d046170", + "Zm00001d014297", + ) + self.assertTrue(result[0]) + self.assertIsNone(result[1]) + + result = eFPUtils.is_efp_input_valid( + "efp_maize", "maize_iplant", "Compare", "Zm00001d046170", "Abc" + ) + expected = "Gene 2 is invalid." + self.assertFalse(result[0]) + self.assertEqual(result[1], expected) + + # Test if both gene are the same in eFP Comare mode + result = eFPUtils.is_efp_input_valid( + "efp_arachis", + "Arachis_Atlas", + "Compare", + "Adur10002_comp0_c0_seq1", + "Adur10002_comp0_c0_seq1", + ) + expected = "In compare mode, both genes should be different." + self.assertFalse(result[0]) + self.assertEqual(result[1], expected)