search nuclei via id and seg id + caching

neuvue_project/neuvue/settings.py

@@ -204,12 +204,16 @@
 # Neuvue Specific Settings
 NEUVUE_QUEUE_ADDR = "https://queue.neuvue.io/"
 SANDBOX_ID = '6269888a101fc4da81fdd410'
-# NEUVUE_QUEUE_ADDR = "http://localhost:9005"
 NEUVUE_CLIENT_SETTINGS = {
     # "local" : True
 }
-NUCLEUS_NUERON_SVM = 'nucleus_neuron_svm'
-CELL_CLASS_MODEL = 'allen_soma_coarse_cell_class_model_v2'
+
+# Annotation Tables
+NEURON_TABLE = 'nucleus_neuron_svm'
+CELL_CLASS_TABLE = 'allen_soma_coarse_cell_class_model_v2'
+DAYS_UNTIL_EXPIRED = 3
+CACHED_TABLES_PATH = os.path.join(STATIC_ROOT, "tables")
+
 # Task Timeout in Seconds
 TIMEOUT = 900
 

neuvue_project/neuvue/urls.py

@@ -65,7 +65,7 @@
     path('synapse/<str:root_ids>', SynapseView.as_view(), name="synapse"),
     path('synapse/<str:root_ids>/<str:pre_synapses>/<str:post_synapses>/<str:cleft_layer>/<str:timestamp>', SynapseView.as_view(), name="synapse"),
     path('nuclei/', NucleiView.as_view(), name="nuclei"),
-    path('nuclei/<str:nuclei_ids>', NucleiView.as_view(), name="nuclei"),
+    path('nuclei/<str:given_ids>', NucleiView.as_view(), name="nuclei"),
     path('report/', ReportView.as_view(), name="report"),
     path('userNamespace/', UserNamespaceView.as_view(), name="user-namespace"),
     path('save_state', SaveStateView.as_view(), name="save-state"),

neuvue_project/templates/nuclei.html

@@ -4,15 +4,15 @@
 
 {% block content %}
 
-{% if not nuclei_ids or error %}
+{% if not given_ids or error %}
 
 <div class="basic workspace">
   <div class="inspect-container">
     <form id="mainForm" action="" method="post" onSubmit="triggerLoadingSpinner('submit-spinner')">
       {% csrf_token %}
       <div class="form-group">
-        <label class="text-white-50" for="rootIDInput">Nuclei ID (Enter Nuclei ID (s) separated by commas)</label>
-        <input class="form-control" id="rootIDInput" name="nuclei_ids" required="true">
+        <label class="text-white-50" for="rootIDInput">Nuclei ID and/or Seg ID (Enter ID (s) separated by commas)</label>
+        <input class="form-control" id="rootIDInput" name="given_ids" required="true">
       </div>
       <br>
         <div class="d-flex">
@@ -45,6 +45,9 @@
             <table class="table table-dark table-bordered table-hover">
               {{cell_types|safe}}
             </table>
+            {% if ids_not_found %}
+              <span> <code> IDs not found: </code>{{ids_not_found}}</span>
+            {% endif %}
         </div>
     </div>
     <div id = "instruction-container" class ="sideContentBox" style="padding: 0; border:transparent !important;">

neuvue_project/workspace/neuroglancer.py

@@ -1,18 +1,20 @@
+import time
+import json
+import glob
+import random
+import logging
+from typing import List
+from datetime import datetime, timedelta
+
 from django.conf import settings
 from django.apps import apps
-import pandas as pd
-import numpy as np
-from caveclient import CAVEclient
-from typing import List
-from datetime import datetime
-import json
+
 import requests
 import os
 import backoff
-import random
-from .models import ImageChoices, PcgChoices
-
-
+import pandas as pd
+import numpy as np
+from caveclient import CAVEclient
 from nglui.statebuilder import (
     ImageLayerConfig, 
     SegmentationLayerConfig, 
@@ -23,14 +25,50 @@
     ChainedStateBuilder
     )
 
-from .models import Namespace, NeuroglancerLinkType, PcgChoices
+from .models import Namespace, NeuroglancerLinkType, PcgChoices, ImageChoices
+
 
-import logging
 logging.basicConfig(level=logging.DEBUG)
 logger = logging.getLogger(__name__)
 
 Config = apps.get_model('preferences', 'Config')
 
+
+def get_df_from_static(cave_client, table_name):
+
+    @backoff.on_exception(backoff.expo, Exception, max_tries=3)
+    def query_new_table(table_name):
+        logging.info(f'Downloading new table for {table_name}.')
+        df = cave_client.materialize.query_table(table_name)
+        fn = str(round(time.time()))+ '_' + table_name+'.pkl'
+        df.to_pickle(os.path.join(settings.CACHED_TABLES_PATH, fn))
+        return df
+
+    try:
+        if not os.path.exists(settings.CACHED_TABLES_PATH):
+            os.makedirs(settings.CACHED_TABLES_PATH)
+
+        cached_tables = glob.glob(
+            os.path.join(settings.CACHED_TABLES_PATH, '*_'+table_name+'.pkl')
+        )
+        # filter to table of interest
+        if len(cached_tables):
+            file_path = cached_tables[0]
+            file_name = os.path.split(file_path)[1]
+            file_date = int(file_name.split('_')[0])
+            if (datetime.fromtimestamp(file_date) - datetime.fromtimestamp(time.time())) < timedelta(days=settings.DAYS_UNTIL_EXPIRED):
+                logging.info(f'Using cached table for {table_name}.')
+                df = pd.read_pickle(file_path)
+            else:
+                os.remove(file_path)
+                df = query_new_table(table_name)
+        else:
+            df = query_new_table(table_name)
+        return df
+    except Exception: 
+        logger.error('Resource table cannot be queried.')
+        raise Exception(f'Table {table_name} unavailable.')
+
 def create_base_state(seg_ids, coordinate, namespace=None):
     """Generates a base state containing imagery and segmentation layers. 
 
@@ -323,11 +361,11 @@ def _get_soma_center(root_ids: List, cave_client):
         array: array for the position of the soma
     """
     try:
-        soma_df = cave_client.materialize.query_table(settings.NUCLEUS_NUERON_SVM, filter_in_dict={
+        soma_df = cave_client.materialize.query_table(settings.NEURON_TABLE, filter_in_dict={
             'pt_root_id': root_ids[:3]
         })
         if not len(soma_df):
-            soma_df = cave_client.materialize.query_table(settings.NUCLEUS_NUERON_SVM, filter_in_dict={
+            soma_df = cave_client.materialize.query_table(settings.NEURON_TABLE, filter_in_dict={
             'pt_root_id': root_ids
             })
         if len(soma_df) > 3:
@@ -672,30 +710,34 @@ def construct_synapse_state(root_ids: List, flags: dict = None):
         })
     return json.dumps(state_dict), synapse_stats
 
-def construct_nuclei_state(nuclei_ids: List):
+def construct_nuclei_state(given_ids: List):
     """Construct state for the synapse viewer.
 
     Args:
-        root_ids (list): segment root id
-        flags (dict): query parameters
-            - pre_synapses
-            - post_synapses
-            - cleft_layer
-            - timestamp
+        given_ids (list): nuclei and/or pt_root_ids
 
     Returns:
         string: json-formatted state
         dict: synapse stats
     """
+    given_ids = [int(x) for x in given_ids]
     cave_client = CAVEclient('minnie65_phase3_v1', auth_token=os.environ['CAVECLIENT_TOKEN'])
-    soma_df = cave_client.materialize.query_table(settings.NUCLEUS_NUERON_SVM, filter_in_dict={
-            'id': nuclei_ids
-            })
 
+
+    soma_df = get_df_from_static(cave_client, settings.NEURON_TABLE)
+    soma_df = soma_df[(soma_df.id.isin(given_ids))|(soma_df.pt_root_id.isin(given_ids))]
+
+    # identify inputs that were not found in the table and format to display to user
+    ids_not_found = list(set(given_ids) - set().union(soma_df.id,soma_df.pt_root_id))
+    formatted_not_found_ids = ', '.join([str(id) for id in ids_not_found]) if len(ids_not_found) else ''
+
     root_ids = soma_df['pt_root_id'].values
     nuclei_points = np.array(soma_df['pt_position'].values)
     position = nuclei_points[0] if len(nuclei_points) else [] # check what happens when bad values are returned -- add an error case
 
+    if not len(root_ids):
+        raise Exception("ID is outdated or does not exist.")
+
     data_list = [None]
     base_state = create_base_state(root_ids, position)
 
@@ -712,17 +754,17 @@ def get_cell_type(nuclei_id, cell_class_df):
             cell_type = filtered_row.cell_type.values[0] if len(filtered_row) else "NaN"
             return cell_type
 
-        cell_class_info_df = cave_client.materialize.query_table(settings.CELL_CLASS_MODEL, filter_in_dict={
-                'id': soma_df.id.values
-                })
+        cell_class_info_df = get_df_from_static(cave_client, settings.CELL_CLASS_TABLE)
+        cell_class_info_df = cell_class_info_df[cell_class_info_df.id.isin(soma_df.id.values)]
+
 
         updated_soma_df = pd.merge(soma_df, cell_class_info_df, on='id', how='outer')
         updated_soma_df.cell_type_y = updated_soma_df.cell_type_y.fillna('unknown')
 
-        type_table = '<thead><tr><th>Nuclei ID</th><th>Type</th></tr></thead><tbody>'
-        for nucleus_id in updated_soma_df.id.values:
+        type_table = '<thead><tr><th>Nuclei ID</th><th>Seg ID</th><th>Type</th></tr></thead><tbody>'
+        for nucleus_id, seg_id in zip(updated_soma_df.id.values, updated_soma_df.pt_root_id_x.values):
             cell_type = get_cell_type(nucleus_id, cell_class_info_df)
-            type_table += '<tr><td>'+str(nucleus_id)+'</td><td>'+cell_type+'</td><tr>'
+            type_table += '<tr><td>'+str(nucleus_id)+'</td><td>'+str(seg_id)+'</td><td>'+cell_type+'</td><tr>'
         type_table += '</tbody>'
 
         return type_table, updated_soma_df
@@ -742,7 +784,7 @@ def get_cell_type(nuclei_id, cell_class_df):
     state_dict["selectedLayer"] = {"layer": "seg", "visible": True}
     state_dict['jsonStateServer'] = settings.JSON_STATE_SERVER
 
-    return json.dumps(state_dict), cell_type_table
+    return json.dumps(state_dict), cell_type_table, formatted_not_found_ids
 
 
 def refresh_ids(ng_state:str, namespace:str): 

neuvue_project/workspace/views/tools.py

@@ -210,41 +210,37 @@ def post(self, request, *args, **kwargs):
 
 
 class NucleiView(View):
-    def get(self, request, nuclei_ids=None, *args, **kwargs):
+    def get(self, request, given_ids=None, *args, **kwargs):
         if not is_authorized(request.user):
-            logging.warning(f"Unauthorized requests from {request.user}.")
-            return redirect(reverse("index"))
+            logging.warning(f'Unauthorized requests from {request.user}.')
+            return redirect(reverse('index'))
 
-        if nuclei_ids in settings.STATIC_NG_FILES:
-            return redirect(
-                f"/static/workspace/{nuclei_ids}", content_type="application/javascript"
-            )
+        if given_ids in settings.STATIC_NG_FILES:
+            return redirect(f'/static/workspace/{given_ids}', content_type='application/javascript')
 
-        context = {"nuclei_ids": None, "error": None}
+        context = {
+            "given_ids": None,
+            "error": None
+        }
 
-        if nuclei_ids is None:
+        if given_ids is None:
             return render(request, "nuclei.html", context)
 
-        nuclei_ids = [x.strip() for x in nuclei_ids.split(",")]
+        given_ids = [x.strip() for x in given_ids.split(',')]
 
         try:
-            context["nuclei_ids"] = nuclei_ids
-            context["ng_state"], context["cell_types"] = construct_nuclei_state(
-                nuclei_ids=nuclei_ids
-            )
-        except Exception as e:
-            context["error"] = e
+            context['given_ids'] = given_ids
+            context['ng_state'], context['cell_types'], context['ids_not_found'] = construct_nuclei_state(given_ids=given_ids)
+        except Exception as e: 
+            context['error'] = e
 
         return render(request, "nuclei.html", context)
 
+
     def post(self, request, *args, **kwargs):
-        nuclei_ids = request.POST.get("nuclei_ids")
+        given_ids = request.POST.get("given_ids")
+
+        return redirect(reverse('nuclei', kwargs={
+            "given_ids": given_ids,
+        }))
 
-        return redirect(
-            reverse(
-                "nuclei",
-                kwargs={
-                    "nuclei_ids": nuclei_ids,
-                },
-            )
-        )