Merge pull request #472 from AndersenLab/enhancement/phenotype-datast…

…ore-multiple-name-fields Render trait display names everywhere
AndersenLab · Apr 18, 2024 · eb23b3f · eb23b3f
2 parents 9ced4f8 + 75e9873
commit eb23b3f
Show file tree

Hide file tree

Showing 24 changed files with 474 additions and 194 deletions.
diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -322,6 +322,28 @@
                 "SITE_BASE_URL": "https://localhost:8080"
             }
         },
+        {
+            "name": "Run POPULATE_PHENOTYPES_DATASTORE",
+            "type": "python",
+            "request": "launch",
+            "program": "${workspaceFolder}/src/modules/db_operations/main.py",
+            "console": "integratedTerminal",
+            "python": "${workspaceFolder}/src/modules/db_operations/venv/bin/python",
+            "justMyCode": true,
+            "envFile": "${workspaceFolder}/src/modules/db_operations/.env",
+            "env": {
+                "DATABASE_OPERATION": "POPULATE_PHENOTYPES_DATASTORE",
+                "USE_MEMOIZATION": "1",
+                // "MODULE_DB_OPERATIONS_CONNECTION_TYPE": "localhost",
+                // "MODULE_DB_OPERATIONS_CONNECTION_TYPE": "memory",
+                "MODULE_DB_OPERATIONS_CONNECTION_TYPE": "file",
+                "MODULE_DB_OPERATIONS_CONNECTION_FILE": "${workspaceFolder}/temp/local.db",
+                "MODULE_DB_TIMEOUT": "300",
+                "CAENDR_USE_SSL": "1",
+                "SENTRY_URL": "",
+                "SITE_BASE_URL": "https://localhost:8080",
+            },
+        },
         {
             "name": "DB_OP TEST_ECHO",
             "type": "python",
@@ -348,6 +370,24 @@
                 "SITE_BASE_URL": "https://localhost:8080",
             }
         },
+        {
+            "name": "DbOp: Recompute Phenotype Report Cached Names",
+            "type": "python",
+            "request": "launch",
+            "program": "${workspaceFolder}/src/modules/db_operations/main.py",
+            "console": "integratedTerminal",
+            "python": "${workspaceFolder}/src/modules/db_operations/venv/bin/python",
+            "justMyCode": true,
+            "envFile": "${workspaceFolder}/src/modules/db_operations/.env",
+            "env": {
+                "DATABASE_OPERATION": "RECOMPUTE_PHENOTYPE_REPORT_CACHED_NAMES",
+                "RELOAD_FILES": "false",
+                "MODULE_DB_TIMEOUT": "300",
+                "CAENDR_USE_SSL": "1",
+                "SENTRY_URL": "",
+                "SITE_BASE_URL": "https://localhost:8080",
+            }
+        },
         {
             "name": "Test Indel Finder",
             "type": "python",

diff --git a/src/modules/db_operations/operations.py b/src/modules/db_operations/operations.py
@@ -2,7 +2,7 @@
 from caendr.services.cloud.postgresql import health_database_status
 from caendr.services.logger import logger
 
-from caendr.models.datastore import Species
+from caendr.models.datastore import Species, PhenotypeReport
 from caendr.models.sql import DbOp, WormbaseGene, WormbaseGeneSummary, Strain, StrainAnnotatedVariant, PhenotypeDatabase, PhenotypeMetadata
 from caendr.services.sql.db import backup_external_db
 from caendr.services.sql.etl import ETLManager
@@ -49,6 +49,9 @@ def execute_operation(app, db, db_op: DbOp, species=None, reload_files=True):
     logger.info("Using MOCK DATA")
     drop_and_populate_all_tables(app, db, species)
 
+  elif db_op == DbOp.RECOMPUTE_PHENOTYPE_REPORT_CACHED_NAMES:
+    PhenotypeReport.recompute_cached_display_names()
+
 
 
 def drop_and_populate_strains(app, db, species, reload_files=True):

diff --git a/src/modules/db_operations/seed_trait_files.py b/src/modules/db_operations/seed_trait_files.py
@@ -2,9 +2,9 @@
 from caendr.services.cloud.secret  import get_secret
 
 from caendr.models.datastore       import TraitFile
-from caendr.models.error           import NotFoundError, NonUniqueEntity
+from caendr.models.error           import NotFoundError, NonUniqueEntity, GoogleSheetsParseError
 from caendr.models.status          import PublishStatus
-from caendr.services.cloud.sheets  import get_field_from_record
+from caendr.services.cloud.sheets  import check_missing_columns, get_field_from_record
 from caendr.utils.data             import unique_id
 from caendr.utils.local_files      import LocalGoogleSheet
 
@@ -13,6 +13,31 @@
 ANDERSEN_LAB_TRAIT_SHEET = get_secret(f'ANDERSEN_LAB_TRAIT_SHEET')
 
 
+# Set of column headers that must exist in the sheet
+# NOTE: Not every entry needs to define all of these fields -- these are just the columns that should be available in the sheet.
+REQUIRED_SHEET_HEADERS = frozenset({
+  'Trait_Name_CaeNDR',
+  'Trait_Name_User',
+  'Species',
+  'Trait_Name_Display1',
+  'Trait_Name_Display2',
+  'Trait_Name_Display3',
+  'Short_Description',
+  'Long_Description',
+  'Units',
+  'Publication',
+  'Protocol',
+  'Institution',
+  'Captured_By_UserID',
+  'Capture_Date',
+
+  # TODO: How should we handle category tags?
+  # 'Category1',
+  # 'Category2',
+  # 'Category3',
+})
+
+
 
 def populate_andersenlab_trait_files():
   '''
@@ -27,7 +52,17 @@ def populate_andersenlab_trait_files():
 
   # Fetch the Google Sheet and loop through all records
   trait_sheet = LocalGoogleSheet( 'TRAITS', ANDERSEN_LAB_TRAIT_SHEET )
-  for record in trait_sheet.fetch_resource().get_all_records():
+  resource = trait_sheet.fetch_resource()
+
+  # Make sure the Google Sheet has all the required columns
+  missing_columns = check_missing_columns(resource, REQUIRED_SHEET_HEADERS)
+  if len(missing_columns):
+    msg = f'{trait_sheet} (ID: {trait_sheet._sheet_id}) is missing columns: [{", ".join(missing_columns)}]'
+    logger.error(msg)
+    raise GoogleSheetsParseError(msg)
+
+  # Loop through rows in the sheet
+  for record in resource.get_all_records():
 
     # Require that unique CaeNDR trait name is defined
     trait_unique_name = record.get('Trait_Name_CaeNDR')
@@ -61,9 +96,12 @@ def populate_andersenlab_trait_files():
         'species':           get_field_from_record(record, 'Species', nullable=False),
 
         # About trait
-        'description_short': get_field_from_record(record, 'Short_Description'),
-        'description_long':  get_field_from_record(record, 'Long_Description'),
-        'units':             get_field_from_record(record, 'Units'),
+        'trait_name_display_1': get_field_from_record(record, 'Trait_Name_Display1'),
+        'trait_name_display_2': get_field_from_record(record, 'Trait_Name_Display2'),
+        'trait_name_display_3': get_field_from_record(record, 'Trait_Name_Display3'),
+        'description_short':    get_field_from_record(record, 'Short_Description'),
+        'description_long':     get_field_from_record(record, 'Long_Description'),
+        'units':                get_field_from_record(record, 'Units'),
 
         # Tag list (categories)
         'tags': list(filter(

diff --git a/src/modules/site-v2/base/static/js/plot.js b/src/modules/site-v2/base/static/js/plot.js
@@ -204,7 +204,7 @@ function render_scatterplot_histograms(container_selector, data, config={}) {
         .attr('x', margin.left + (width / 2))
         .attr('y', height + margin.top + hist_height + 36)
         .attr('text-anchor', 'middle')
-        .text(config['x_label'])
+        .html(config['x_label'])
     }
 
     // Add label for y-axis, if one is provided
@@ -216,7 +216,7 @@ function render_scatterplot_histograms(container_selector, data, config={}) {
         .attr('y', 0)
         .attr('dy', '.75em')
         .attr('text-anchor', 'middle')
-        .text(config['y_label'])
+        .html(config['y_label'])
 
       }
 
@@ -349,7 +349,7 @@ function render_histogram(container_selector, data, config={}) {
       .attr('x', margin.left + (width / 2))
       .attr('y', height + margin.top + 36)
       .attr('text-anchor', 'middle')
-      .text(config['x_label'])
+      .html(config['x_label'])
   }
 
   // Add the histogram
@@ -501,7 +501,7 @@ function render_ranked_barplot(container_selector, data, config={}) {
       .attr('y', 0)
       .attr('dy', '.75em')
       .attr('text-anchor', 'middle')
-      .text(config['y_label'])
+      .html(config['y_label'])
     }
     //Render tooltips
     setTooltips();

diff --git a/src/modules/site-v2/base/views/tools/genetic_mapping.py b/src/modules/site-v2/base/views/tools/genetic_mapping.py
@@ -33,24 +33,6 @@
 
 
 
-def results_columns():
-  return [
-    {
-      'title': 'Description',
-      'class': 'label',
-      'field': 'label',
-      'width': 0.6,
-      'link_to_data': True,
-    },
-    {
-      'title': 'Trait',
-      'class': 'trait',
-      'field': 'trait',
-      'width': 0.4,
-    },
-  ]
-
-
 @genetic_mapping_bp.route('', methods=['GET'])
 @jwt_required()
 def genetic_mapping():
@@ -162,7 +144,6 @@ def list_results():
     # Table info
     'species_list': Species.all(),
     'items': list_reports(NemascanReport, None if show_all else user, filter_errs),
-    'columns': results_columns(),
 
     'JobStatus': JobStatus,
   })

diff --git a/src/modules/site-v2/base/views/tools/heritability_calculator.py b/src/modules/site-v2/base/views/tools/heritability_calculator.py
@@ -51,23 +51,6 @@
 )
 
 
-def results_columns():
-  return [
-    {
-      'title': 'Description',
-      'class': 'label',
-      'field': 'label',
-      'width': 0.6,
-      'link_to_data': True,
-    },
-    {
-      'title': 'Trait',
-      'class': 'trait',
-      'field': 'trait',
-      'width': 0.4,
-    },
-  ]
-
 
 @heritability_calculator_bp.route('')
 @jwt_required()
@@ -136,7 +119,6 @@ def list_results():
     # Table info
     'species_list': Species.all(),
     'items': list_reports(HeritabilityReport, None if show_all else user, filter_errs),
-    'columns': results_columns(),
 
     'JobStatus': JobStatus,
   })

diff --git a/src/modules/site-v2/base/views/tools/pairwise_indel_finder.py b/src/modules/site-v2/base/views/tools/pairwise_indel_finder.py
@@ -32,30 +32,6 @@
 
 
 
-def results_columns():
-  return [
-    {
-      'title': 'Site',
-      'class': 'site',
-      'field': 'site',
-      'width': 0.5,
-      'link_to_data': True,
-      'data_order': lambda e: e['site'],
-    },
-    {
-      'title': 'Strain 1',
-      'class': 's1',
-      'field': 'strain_1',
-      'width': 0.25,
-    },
-    {
-      'title': 'Strain 2',
-      'class': 's2',
-      'field': 'strain_2',
-      'width': 0.25,
-    },
-  ]
-
 def try_get_sv_strains(species):
   try:
     return get_sv_strains(species)
@@ -185,7 +161,6 @@ def list_results():
     # Table info
     'species_list': Species.all(),
     'items': list_reports(IndelPrimerReport, None if show_all else user, filter_errs),
-    'columns': results_columns(),
 
     'JobStatus': JobStatus,
   })

diff --git a/src/modules/site-v2/base/views/tools/phenotype_database.py b/src/modules/site-v2/base/views/tools/phenotype_database.py
@@ -44,31 +44,6 @@ def check_bp_enabled():
 
 
 
-def results_columns():
-  return [
-    {
-      'title': 'Description',
-      'class': 'label',
-      'field': 'label',
-      'width': 0.2,
-      'link_to_data': True,
-    },
-    {
-      'title': 'Trait 1',
-      'class': 's1',
-      'field': 'trait_1_name',
-      'width': 0.4,
-    },
-    {
-      'title': 'Trait 2',
-      'class': 's2',
-      'field': 'trait_2_name',
-      'width': 0.4,
-    },
-  ]
-
-
-
 #
 # Main Endpoint
 #
@@ -253,7 +228,7 @@ def submit():
   # Read & clean fields from JSON data
   data = {
     field: bleach.clean(request.json.get(field))
-      for field in {'label', 'species', 'trait_1', 'trait_1_dataset'}
+      for field in {'species', 'trait_1', 'trait_1_dataset'}
   }
 
   # Read & clean values for trait 2, if given
@@ -312,7 +287,6 @@ def list_results():
     # Table info
     'species_list': Species.all(),
     'items': list_reports(PhenotypeReport, user = None if show_all else user, filter_errs=filter_errs),
-    'columns': results_columns(),
 
     'JobStatus': JobStatus,
   })

diff --git a/src/modules/site-v2/templates/_includes/macros.html b/src/modules/site-v2/templates/_includes/macros.html
@@ -167,8 +167,3 @@
   <em>Unable to retrieve files.</em>
 {%- endif %}
 {%- endmacro %}
-
-
-{%- macro label_string_units(name, units=null) %}
-{%- if units -%} "{{ name }} ({{ units }})" {%- else -%} "{{ name }}" {%- endif -%}
-{%- endmacro %}