server refactor (#1140)

This PR contains a refactoring to make adding new features easier.

The new features include supporting the tiledb format, and the multi dataset application.

The refactoring includes

Simplifying the directory structure and files.
a class structure to handle annotations (currently one type: AnnotationsLocalFile).
a class to handle application configuration
a class structure to handle matrix data (currently AnndataAdaptor and CxgAdaptor). CxgAdaptor uses tiledb.
Algorithms that were previously dependent on the scanpy anndata object are now generalized to work with an abstract interface.
The multi dataset option is not fully supported yet, and so the option to use it is hidden.
Use "cli launch --dataroot ..."
To access this feature.

All combinations of app single dataset/ app multi dataset and AnndataAdaptor/CxgAdaptor work with all the features, such as annotations, ontologies, diffexp.

.gitignore

@@ -17,12 +17,12 @@ venv/
 cellxgene/
 
 # client build
-server/app/web/static/css/
-server/app/web/static/img/
-server/app/web/static/media/
-server/app/web/static/fonts/
-server/app/web/static/js/
-server/app/web/templates/index\.html
+server/common/web/static/css/
+server/common/web/static/img/
+server/common/web/static/media/
+server/common/web/static/fonts/
+server/common/web/static/js/
+server/common/web/templates/index\.html
 
 # Jupyter Notebook
 .ipynb_checkpoints

MANIFEST.in

@@ -1,5 +1,5 @@
-recursive-include server/app/web/templates *
-recursive-include server/app/web/static *
+recursive-include server/common/web/templates *
+recursive-include server/common/web/static *
 
 include server/requirements.txt
 include server/requirements-prepare.txt

Makefile

@@ -34,24 +34,24 @@ build-client:
 build-cli: build-client
 	git ls-files server/ | cpio -pdm $(BUILDDIR)
 	cp -r client/build/  $(CLIENTBUILD)
-	mkdir -p $(SERVERBUILD)/app/web/static/img
-	mkdir -p $(SERVERBUILD)/app/web/templates/
-	cp $(CLIENTBUILD)/index.html $(SERVERBUILD)/app/web/templates/
-	cp -r $(CLIENTBUILD)/static $(SERVERBUILD)/app/web/
-	cp $(CLIENTBUILD)/favicon.png $(SERVERBUILD)/app/web/static/img
-	cp $(CLIENTBUILD)/service-worker.js $(SERVERBUILD)/app/web/static/js/
+	mkdir -p $(SERVERBUILD)/common/web/static/img
+	mkdir -p $(SERVERBUILD)/common/web/templates/
+	cp $(CLIENTBUILD)/index.html $(SERVERBUILD)/common/web/templates/
+	cp -r $(CLIENTBUILD)/static $(SERVERBUILD)/common/web/
+	cp $(CLIENTBUILD)/favicon.png $(SERVERBUILD)/common/web/static/img
+	cp $(CLIENTBUILD)/service-worker.js $(SERVERBUILD)/common/web/static/js/
 	cp MANIFEST.in README.md setup.cfg setup.py $(BUILDDIR)
 
 # If you are actively developing in the server folder use this, dirties the source tree
 .PHONY: build-for-server-dev
 build-for-server-dev: clean-server build-client
-	mkdir -p server/app/web/static/img
-	mkdir -p server/app/web/static/js
-	mkdir -p server/app/web/templates/
-	cp client/build/index.html server/app/web/templates/
-	cp -r client/build/static server/app/web/
-	cp client/build/favicon.png server/app/web/static/img
-	cp client/build/service-worker.js server/app/web/static/js/
+	mkdir -p server/common/web/static/img
+	mkdir -p server/common/web/static/js
+	mkdir -p server/common/web/templates/
+	cp client/build/index.html server/common/web/templates/
+	cp -r client/build/static server/common/web/
+	cp client/build/favicon.png server/common/web/static/img
+	cp client/build/service-worker.js server/common/web/static/js/
 
 
 # TESTING
@@ -137,10 +137,6 @@ dev-env-client:
 dev-env-server:
 	pip install -r server/requirements-dev.txt
 
-.PHONY: gui-env
-gui-env: dev-env
-	pip install -r server/requirements-gui.txt
-
 # give PART=[major, minor, part] as param to make bump
 .PHONY: bump
 bump:
@@ -195,30 +191,3 @@ install-dist: uninstall
 uninstall:
 	pip uninstall -y cellxgene || :
 
-
-# GUI
-
-.PHONY: build-assets
-build-assets:
-	pyside2-rcc server/gui/cellxgene.qrc -o server/gui/cellxgene_rc.py
-
-.PHONY: gui-spec-osx
-gui-spec-osx: clean-lite gui-env
-	pip install -e .[gui]
-	pyi-makespec -D -w --additional-hooks-dir server/gui/ -n cellxgene  --add-binary='/System/Library/Frameworks/Tk.framework/Tk':'tk' --add-binary='/System/Library/Frameworks/Tcl.framework/Tcl':'tcl'  --add-data server/app/web/templates/:server/app/web/templates/ --add-data server/app/web/static/:server/app/web/static/ --icon server/gui/images/cxg_icons.icns server/gui/main.py
-	mv cellxgene.spec cellxgene-osx.spec
-
-.PHONY: gui-spec-windows
-gui-spec-windows: clean-lite dev-env
-	pip install -e .[gui]
-	pyi-makespec -D -w --additional-hooks-dir server/gui/ -n cellxgene --add-data server/app/web/templates;server/app/web/templates --add-data server/app/web/static;server/app/web/static --icon server/gui/images/icon.ico server/gui/main.py
-	mv cellxgene.spec cellxgene-windows.spec
-
-.PHONY: gui-build-osx
-gui-build-osx: clean-lite
-	pyinstaller --clean cellxgene-osx.spec
-
-.PHONY: gui-build-windows
-gui-build-windows: clean-lite
-	pyinstaller --clean cellxgene-windows.spec
-

dev_docs/developer_scripts.md

@@ -43,7 +43,6 @@ Installs requirements files
 
 ```
 dev-env - installs requirements and requirments-dev (for building code)
-gui-env - installs requirements and requirments-dev and requirments-gui (for building native app)
 ```
 
 ### install commands
@@ -59,17 +58,6 @@ install-dist - installs from local dist folder
 uninstall - uninstalls cellxgene
 ```
 
-### gui
-
-Commands for building the native app
-
-```
-build-assets - builds the image and icon assets for the gui to pull from
-gui-spec-osx - creates the initial spec file for osx, do not run unless you are starting from scratch, one time only
-gui-spec-windows - creates the initial spec file for windows, do not run unless you are starting from scratch, one time only
-gui-build-osx - builds the app from the osx spec file
-gui-build-windows - builds the app from the windows spec file
-```
 ## Client Makefile
 
 The following phony `make` targets in `client/Makefile` are convenience methods for getting you up and developing.

pyproject.toml

@@ -19,7 +19,6 @@ exclude = '''
     | dist
     | server/app/util/fbs/NetEncoding
   )/
-  | server/gui/cellxgene_rc.py
 
 )
 '''

server/Makefile

@@ -2,8 +2,8 @@ include ../common.mk
 
 .PHONY: clean
 clean:
-	rm -f app/web/templates/index.html
-	rm -rf app/web/static
+	rm -f common/web/templates/index.html
+	rm -rf common/web/static
 
 .PHONY: unit-test
 unit-test: