Make GNN plots available as an SPT CLI command and API endpoint (#320)

* move into spt/graphs * sketch CLI * rename fraction plotter * rely on api server implementation for now * accommodate configreader's stripped down toml support * rename db upload_sync * add gnn plot api endpoint * hotfix typos, missing import * add gnn plot config table * finish apiserver endpoint * change upload-sync-small to upload a config file string * have fastapi handle img format validation * remove unused file * Update test artifact * remove weird thread pool stuff * hotfix config reading when calling from apiserver * update analysis_replication for gnn plots * Version bump * Remove some build dependencies --------- Co-authored-by: James Mathews <mail@jmath.read-books.org>
nadeemlab · May 15, 2024 · f1ce820 · f1ce820
1 parent fcf5342
commit f1ce820
Show file tree

Hide file tree

Showing 28 changed files with 739 additions and 292 deletions.
diff --git a/analysis_replication/README.md b/analysis_replication/README.md
@@ -30,7 +30,7 @@ substituting the argument with the address of your local API server. (See *Setti
 - You can alternatively store the API host in `api_host.txt` and omit the command-line argument above.
 - The run result is here in [results.txt](results.txt).
 
-# Cell arrangment figure generation
+## Cell arrangement figure generation
 
 One figure is generated programmatically from published source TIFF files.
 To run the figure generation script, alter the command below to reference your own database configuration file and path to unzipped Moldoveanu et al dataset.
@@ -39,11 +39,11 @@ To run the figure generation script, alter the command below to reference your o
 python retrieve_example_plot.py dataset_directory/ ~/.spt_db.config
 ```
 
-# GNN importance fractions figure generation
+## GNN importance fractions figure generation
 
-Another figure is generated programmatically from extractions from Graph Neural Network models, provided by the API.
+This plot replication requires the installation of SPT, as it's not a script in this directory. Instead, it's a command in the `spt graphs` CLI that uses the configuration files stored in `gnn_figure/` to reproduce the plots seen in our publication.
 
 ```bash
-cd gnn_figure/
-python graph_plugin_plots.py
+spt graphs plot-importance-fractions --config_path gnn_figure/melanoma_intralesional_il2.config --output_filename gnn_figure/melanoma_intralesional_il2.png
+spt graphs plot-importance-fractions --config_path gnn_figure/urothelial_ici.config --output_filename gnn_figure/urothelial_ici.png
 ```
diff --git a/analysis_replication/gnn_figure/melanoma_intralesional_il2.config b/analysis_replication/gnn_figure/melanoma_intralesional_il2.config
@@ -0,0 +1,15 @@
+[general]
+study_name = Melanoma intralesional IL2
+
+[plot-importance-fractions]
+host_name = http://oncopathtk.org/api
+phenotypes = Tumor, Adipocyte or Langerhans cell, Natural killer cell, CD4+ T cell, Nerve, B cell, CD4+/CD8+ T cell, CD4+ regulatory T cell, CD8+ natural killer T cell, CD8+ regulatory T cell, CD8+ T cell, Double negative regulatory T cell, T cell/null phenotype, Natural killer T cell, CD4+ natural killer T cell
+plugins = cg-gnn, graph-transformer
+figure_size = 11, 8
+orientation = horizontal
+[plot-importance-fractions.cohort0]
+index_int = 1
+label = Non-responder
+[plot-importance-fractions.cohort1]
+index_int = 3
+label = Responder
diff --git a/analysis_replication/gnn_figure/melanoma_intralesional_il2.json b/analysis_replication/gnn_figure/melanoma_intralesional_il2.json
diff --git a/analysis_replication/gnn_figure/urothelial_ici.config b/analysis_replication/gnn_figure/urothelial_ici.config
@@ -0,0 +1,15 @@
+[general]
+study_name = Urothelial ICI
+
+[plot-importance-fractions]
+host_name = http://oncopathtk.org/api
+phenotypes = Tumor, CD4- CD8- T cell, T cytotoxic cell, T helper cell, Macrophage, intratumoral CD3+ LAG3+, Regulatory T cell
+plugins = cg-gnn, graph-transformer
+figure_size = 14, 5
+orientation = vertical
+[plot-importance-fractions.cohort0]
+index_int = 1
+label = Responder
+[plot-importance-fractions.cohort1]
+index_int = 2
+label = Non-responder
diff --git a/analysis_replication/gnn_figure/urothelial_ici.json b/analysis_replication/gnn_figure/urothelial_ici.json
diff --git a/build/apiserver/Dockerfile b/build/apiserver/Dockerfile
@@ -16,6 +16,7 @@ RUN python -m pip install scikit-learn==1.2.2
 RUN python -m pip install Pillow==9.5.0
 RUN python -m pip install pydantic==2.0.2
 RUN python -m pip install secure==0.3.0
+RUN python -m pip install matplotlib==3.7.1
 ARG version
 ARG service_name
 ARG WHEEL_FILENAME

diff --git a/build/apiserver/Makefile b/build/apiserver/Makefile
@@ -81,4 +81,4 @@ ${TESTS}: setup-testing
 clean:
 >@rm -f ${WHEEL_FILENAME}
 >@rm -f status_code
->@for f in dlogs.db.txt dlogs.api.txt dlogs.od.txt ../../${TEST_LOCATION}\/${MODULE_NAME}/_proximity.json ../../${TEST_LOCATION}\/${MODULE_NAME}/_squidpy.json; do rm -f $$f; done;
+>@for f in dlogs.db.txt dlogs.api.txt dlogs.od.txt ../../${TEST_LOCATION}\/${MODULE_NAME}/_proximity.json ../../${TEST_LOCATION}\/${MODULE_NAME}/_squidpy.json ../../${TEST_LOCATION}\/${MODULE_NAME}/_gnn.svg ; do rm -f $$f; done;
diff --git a/build/build_scripts/.graph_transformer.config b/build/build_scripts/.graph_transformer.config
@@ -0,0 +1,7 @@
+[general]
+db_config_file_path = build/db/.spt_db.config.local
+study_name = Melanoma intralesional IL2
+
+[upload-importances]
+plugin_used = graph-transformer
+datetime_of_run = 2023-10-02 10:46 AM
diff --git a/build/build_scripts/expected_table_counts.txt b/build/build_scripts/expected_table_counts.txt
@@ -9,15 +9,15 @@
                             diagnosis        2
        diagnostic_selection_criterion        4
             expression_quantification    18200
-                feature_specification        1
-                    feature_specifier        4
+                feature_specification        2
+                    feature_specifier        8
                histological_structure      700
 histological_structure_identification      700
          histology_assessment_process        7
                          intervention        2
    plane_coordinates_reference_system        0
                           publication        2
-           quantitative_feature_value      700
+           quantitative_feature_value     1400
                 research_professional       32
                            shape_file      700
           specimen_collection_process        7

diff --git a/build/build_scripts/import_test_dataset1.sh b/build/build_scripts/import_test_dataset1.sh
@@ -10,8 +10,10 @@ nextflow run .
 rm -f .nextflow.log*; rm -rf .nextflow/; rm -f configure.sh; rm -f run.sh; rm -f main.nf; rm -f nextflow.config; rm -rf work/; rm -rf results/
 
 spt graphs upload-importances --config_path=build/build_scripts/.graph.config --importances_csv_path=test/test_data/gnn_importances/1.csv
+spt graphs upload-importances --config_path=build/build_scripts/.graph_transformer.config --importances_csv_path=test/test_data/gnn_importances/1.csv
 
-spt db upload-sync-findings --database-config-file=build/db/.spt_db.config.local test/test_data/findings.json
+spt db upload-sync-small --database-config-file=build/db/.spt_db.config.local findings test/test_data/findings.json
+spt db upload-sync-small --database-config-file=build/db/.spt_db.config.local gnn_plot_configurations test/test_data/gnn_plot.json
 
 spt db status --database-config-file build/db/.spt_db.config.local > table_counts.txt
 diff build/build_scripts/expected_table_counts.txt table_counts.txt

diff --git a/docs/maintenance.md b/docs/maintenance.md
@@ -19,7 +19,9 @@ The modules in this repository are built, tested, and deployed using `make` and
 | [Docker Engine](https://docs.docker.com/engine/install/)   | 20.10.17                         |
 | [Docker Compose](https://docs.docker.com/compose/install/) | 2.10.2                           |
 | [bash](https://www.gnu.org/software/bash/)                 | >= 4                             |
-| [python](https://www.python.org/downloads/)                | >=3.7                            |
+| [python](https://www.python.org/downloads/)                | >=3.7 <3.12                      |
+| [postgresql](https://www.postgresql.org/download/)         | 13.4                             |
+| [toml](https://pypi.org/project/toml/)                     | 0.10.2                           |
 
 A typical development workflow looks like:
 

diff --git a/environment.yml b/environment.yml
@@ -0,0 +1,6 @@
+name: spt
+channels:
+  - conda-forge
+dependencies:
+  - python=3.11
+  - libpq
diff --git a/pyproject.toml.unversioned b/pyproject.toml.unversioned
@@ -31,6 +31,7 @@ repository = "https://github.com/nadeemlab/SPT"
 
 [project.optional-dependencies]
 apiserver = [
+    "matplotlib==3.7.1",
     "fastapi==0.100.0",
     "uvicorn>=0.15.0,<0.16.0",
     "pandas==2.0.2",
@@ -190,7 +191,7 @@ packages = [
     "drop.py",
     "drop_ondemand_computations.py",
     "delete_feature.py",
-    "upload_sync_findings.py",
+    "upload_sync_small.py",
     "collection.py",
 ]
 "spatialprofilingtoolbox.db.data_model" = [
@@ -211,6 +212,7 @@ packages = [
     "extract.py",
     "finalize_graphs.py",
     "generate_graphs.py",
+    "plot_importance_fractions.py",
     "plot_interactives.py",
     "prepare_graph_creation.py",
     "upload_importances.py",