Refactor with fixes, features and improved directory structure (#39)

* 🎨 Apply black formatting
* 🎨⬆️🤠 refactor to use dash-extensions DashBlueprints
* Significantly simplified code base with modular components that are each passed their own page's app (DashBlueprint)
* components and layouts are standardized to use render function that may receive the app with or without arguments
* 🔥 Remove (now) unnecessary index.py
* ⬆️ Pin dash-extensions to first major release
* ⬆️ Pin dash to 2.10
* ⬆️ Add dash-mantine-components
* ⬆️ Add dash-iconify
* 📝 Add contributing docs
* 🎨🐛📝🐳 Fix docs, prop id and docker specs
* 🐳 Add quotes around ports per docker-compose spec
* 📝🎨🔥 add Development resources
🎨🔥 Add icons to not found page and remove unnecessary text
* 🎨 Refactor mag refinement page components
* 🎨 Refactor marker symbols legend to use icons rather than naming shapes
* 🎨 Restyle save selections and settings buttons
* 🎨 Refactor mag-refinement offsettings canvas components
* 🎨 Add minimum example contig_cytoscape component
* 🎨 Add specific backend configurations
* 🎨 Add REDIS_BACKEND details to .env
* 🎨 Add specific backend params to Serverside(..., backend=...) from automappa.data.db import {redis,file_system}_backend
* 📝🔥 Refactor dash components to DashProxy components
* ⬆️ pin dash-mantine-components to 0.12 in env using pip rather than conda
* 🎨 Add icons in page registry with blueprints
* 🎨 Add coverage range slider component
* 🎨 Add Refinement model to track contig cluster refinements
* 🎨 Implement data source to connect MAG refinements table
* 🎨 Add methods to get latest user refinements contig headers
* 🎨 Add interaction b/w coverage range slider and scatterplot2d
* 🎨 Connect download refinements button to data source
* 🎨 Hide MAG refinements switch is now disabled when there are no user refinements that have been made
* 🎨 Refinements table has been styled with new timestamp string and removed 'initial_cluster' column
* 🎨 Refinements table now only retrieves user refinements (i.e. excludes initial clusters)
* 🎨 Add refinements clear button
* Add working data source for completeness/purity boxplot
* 🎨 Add minimal working AgGrid for MAG Summary
* 🎨 Replace dbc.Offcanvas with dmc.Drawer
* 🎨 Replace many of the Dropdowns in Offcanvas with RadioGroup Radio options (less clicks for the user)
* 🥕🐰 Add logic for modular task-queue
* 📝 Add documentation for tasks in CONTRIBUTING and reference tasks README.md
* 🥕 Add minimum working example for task submission, progress display for a task status badge
* 🥕 Add task discovery for home page task in celeryconfig.py
* 🐳🐍 Replace base image of miniforge with mambaforge
* 🥕🎨 Notification system tells user of current state of tasks processing
* 🐛🎨 Move dcc.Store(...) within respective du.Upload component
* 🎨 Add tasks store for getting tasks statuses
* 🐰🥕 Add tasks in home for handling sample data ingestion
* 🎨 Add mag length sum computations
* 🎨 Add MIMAG cluster count computations
* 🎨 Add metadata to sample data card info
* 🐛 Replace na values from cluster col during contig loader pre-processing
* 🎨🔥 Add sample removal feature
* 🎨 Affix settings and save mag buttons

.dockerignore

@@ -3,4 +3,5 @@ test
 Automappa.egg-info
 build
 dist
-.vscode
+.vscode
+.pytest_cache

.env

@@ -1,8 +1,8 @@
 # Required for docker-compose.yml
-SERVER_ROOT_UPLOAD_FOLDER="${HOME}/.automappa/uploads"
+SERVER_ROOT_UPLOAD_FOLDER="/usr/src/app/uploads"
 SERVER_HOST="0.0.0.0"
 SERVER_PORT=8050
-# By, default remove debugging tools --> For development switch this to `True`
+# By default remove debugging tools --> For development switch this to `True`
 SERVER_DEBUG=False
 POSTGRES_USER="admin"
 POSTGRES_PASSWORD="mypass"
@@ -13,6 +13,10 @@ POSTGRES_POOL_PRE_PING=False
 RABBITMQ_DEFAULT_USER="user"
 RABBITMQ_DEFAULT_PASS="pass"
 RABBITMQ_URL="amqp://${RABBITMQ_DEFAULT_USER}:${RABBITMQ_DEFAULT_PASS}@rabbitmq:5672/"
+REDIS_BACKEND_HOST='redis'
+REDIS_BACKEND_PORT='6379'
+REDIS_BACKEND_DB='0'
+REDIS_BACKEND_PASSWORD='RedisPassword'
 CELERY_BACKEND_URL='redis://redis:6379/0'
 CELERY_BROKER_URL="amqp://${RABBITMQ_DEFAULT_USER}:${RABBITMQ_DEFAULT_PASS}@rabbitmq:5672//"
 FLOWER_BROKER_API_URL="http://${RABBITMQ_DEFAULT_USER}:${RABBITMQ_DEFAULT_PASS}@rabbitmq:15672/api"

.gitignore

@@ -11,4 +11,8 @@ build
 Automappa.egg-info
 .env
 data
-db-data
+!automappa/data
+db-data
+file_system_backend
+*.db
+uploads

Dockerfile

@@ -1,20 +1,17 @@
-FROM condaforge/miniforge3:latest
+FROM condaforge/mambaforge:latest
 
-RUN conda install --prune --name base mamba --yes
-
-COPY environment.yml ./environment.yml
-
-RUN mamba env update --name base --file=./environment.yml \
-    && mamba clean --all --force-pkgs-dirs --yes
+COPY environment.yml /tmp/environment.yml
+RUN mamba env update -n base -f /tmp/environment.yml && \
+    mamba clean --all --force-pkgs-dirs --yes
 
 COPY . /usr/src/app
 WORKDIR /usr/src/app
+
 RUN python -m pip install . --ignore-installed --no-deps -vvv
-# Test command is functional
-RUN automappa -h
 
-# Create an unprivileged user for running our Python code.
+# Create an unprivileged user for automappa celery worker
 RUN adduser --disabled-password --gecos '' automappa
+RUN mkdir -p /usr/src/app/uploads && \
+    chown -R automappa:automappa /usr/src/app
 
-# CMD [ "-h" ]
-# ENTRYPOINT [ "automappa" ]
+# CMD ["automappa", "-h"]

Makefile

@@ -45,25 +45,31 @@ endif
 image: Dockerfile
 	docker build . -f $< -t evanrees/automappa:`git branch --show-current`
 
+## Remove automappa-{web,flower,queue} docker images
+rm-images: Dockerfile
+	docker rmi -f `docker images -q automappa-web`
+	docker rmi -f `docker images -q automappa-queue`
+	docker rmi -f `docker images -q automappa-flower`
+
 ## Install automappa entrypoint into current environment
 install: 
 	$(PYTHON_INTERPRETER) -m pip install . --ignore-installed --no-deps -vvv
 
 ## docker compose build from docker-compose.yml
 build: docker-compose.yml
-	docker-compose build
+	docker compose build
 
 ## alias for docker-compose up --always-recreate-deps --remove-orphans --force-recreate
 up: docker-compose.yml
-	docker-compose up --always-recreate-deps --remove-orphans --force-recreate
+	docker compose up --always-recreate-deps --remove-orphans --force-recreate
 
 ## alias for docker-compose down --remove-orphans
 down: docker-compose.yml
-	docker-compose down --remove-orphans
+	docker compose down --remove-orphans
 
 ## alias for docker-compose down --remove-orphans --volumes
 down-v: docker-compose.yml
-	docker-compose down --remove-orphans -v
+	docker compose down --remove-orphans -v
 
 # Run Automappa on test data
 # test: test_data

automappa/Procfile

@@ -1 +1,2 @@
-web: gunicorn index:server
+web: gunicorn app:server --workers 4
+queue: celery --concurrency=2 --app=automappa.tasks.queue worker --loglevel=INFO --uid automappa -E

automappa/__main__.py

@@ -1,11 +1,64 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-from automappa import index
+import argparse
+import logging
 
+from automappa import settings
+from automappa.components import layout
+from automappa.data.database import create_db_and_tables
+from automappa.app import app
 
-def main():
-    index.main()
+logging.basicConfig(
+    format="[%(levelname)s] %(name)s: %(message)s",
+    level=logging.DEBUG,
+)
+
+logger = logging.getLogger(__name__)
+numba_logger = logging.getLogger("numba")
+numba_logger.setLevel(logging.WARNING)
+numba_logger.propagate = False
+h5py_logger = logging.getLogger("h5py")
+h5py_logger.setLevel(logging.WARNING)
+h5py_logger.propagate = False
+root_logger = logging.getLogger()
+root_logger.setLevel(logging.WARNING)
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description="Automappa: An interactive interface for exploration of metagenomes",
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
+    parser.add_argument(
+        "--storage-type",
+        help=(
+            "The type of the web storage. (default: %(default)s)\n"
+            "- memory: only kept in memory, reset on page refresh.\n"
+            "- session: data is cleared once the browser quit.\n"
+            "- local: data is kept after the browser quit.\n"
+        ),
+        choices=["memory", "session", "local"],
+        default="session",
+    )
+    parser.add_argument(
+        "--clear-store-data",
+        help=(
+            "Clear storage data (default: %(default)s)\n"
+            "(only required if using 'session' or 'local' for `--storage-type`)"
+        ),
+        action="store_true",
+        default=False,
+    )
+    args = parser.parse_args()
+
+    create_db_and_tables()
+    app.layout = layout.render(app, args.storage_type, args.clear_store_data)
+    app.run(
+        host=settings.server.host,
+        port=settings.server.port,
+        debug=settings.server.debug,
+    )
 
 
 if __name__ == "__main__":

automappa/app.py

@@ -1,22 +1,29 @@
 import dash_bootstrap_components as dbc
+
+from dash_extensions.enrich import (
+    DashProxy,
+    ServersideOutputTransform,
+)
 import dash_uploader as du
+from automappa.data.database import redis_backend, file_system_backend
+from automappa import settings
 
-from dash_extensions.enrich import DashProxy
-from flask_caching import Cache
-from automappa.settings import server,celery
-# from automappa.tasks import long_callback_manager
 
 app = DashProxy(
     name=__name__,
     title="Automappa",
     external_stylesheets=[dbc.themes.LUX, dbc.icons.BOOTSTRAP],
     update_title="Automapping...",
     suppress_callback_exceptions=True,
-    # long_callback_manager=long_callback_manager,
+    prevent_initial_callbacks=False,
+    use_pages=True,
+    pages_folder="",
+    transforms=[
+        ServersideOutputTransform(
+            default_backend=[file_system_backend],
+            backends=[redis_backend, file_system_backend],
+        ),
+    ],
 )
-cache = Cache(app.server, config={
-    # try 'filesystem' if you don't want to setup redis
-    'CACHE_TYPE': 'redis',
-    'CACHE_REDIS_URL': celery.backend_url
-})
-du.configure_upload(app=app, folder=server.root_upload_folder)
+
+du.configure_upload(app=app, folder=settings.server.root_upload_folder)