Merge pull request #147 from AllenInstitute/feature-channel_timestamps

Fix for Pamela's Barcode Paths

scripts/cron.py

@@ -100,12 +100,27 @@
             python_path + " -d " + utils_dir + " -v " + proj_dandi_value
         print(shlex.split(cmd))
         subprocess.call(shlex.split(cmd))
+        fb.update_session_status(project, session, "Conversion Running")        
+
+    # Trigger uploads for sessions with upload status
+    print("List of sessions to upload")
+    upload_list = fb.get_dandi_statuses()
+    for session in upload_list:
+        dandi_file = fb.view_session(project, sess)['allen']
+        match = re.search(r'specimen_(\d+)', fb.view_session(project, sess)['path'])
+        if match:
+            specimen_number = match.group(1)
+        with open("dandi_ephys_uploads.py") as upload:
+            code = compile(upload.read(), "dandi_ephys_uploads.py", "exec")
+            exec(code, {"dandi_val": proj_dandi_value, "sess_id":session, "dandi_file": dandi_file, "subject_id": specimen_number})
+    dandi.find_dandiset_sessions(project, proj_dandi_value)
         fb.update_session_status(project, session, "Conversion Running")
 
     # Update the dandi locations for the project
     dandi.find_dandiset_sessions(project, proj_dandi_value)
 
 
+
 for project in o_proj_list:
     current_dir = os.path.abspath(__file__)
     current_dir = current_dir.replace('scripts/cron.py', '')
@@ -204,6 +219,4 @@
             subprocess.run(shlex.split(cmd))
             fb.update_session_status(
                 project, session, "Raw Conversion Running")
-
-    # Update the dandi locations for the project
     dandi.find_dandiset_sessions(project, proj_dandi_value)

src/openscopenwb/ecephys_modules.py

@@ -304,15 +304,16 @@ def ecephys_align_timestamps(module_params):
     })
     print("File was found: " + str(file_found))
     if (file_found):
+        channel_states_path = glob(join(module_params['base_directory'],
+                                        '**/**' + probe_idx, '**/**', "*channel_states.npy"), recursive=True)[0]
+        timestamp_path = glob(join(module_params['base_directory'],
+                                        '**/**' + probe_idx, '**/**', "*stamps.npy"), recursive=True)[0]
         probe_dict = {
             'name': probe_idx,
             'sampling_rate': 30000.,
             'lfp_sampling_rate': 2500.,
-            'barcode_channel_states_path': join(events_directory,
-                                                'channel_states.npy'),
-            'barcode_timestamps_path': join(
-                events_directory,
-                'event_timestamps.npy'),
+            'barcode_channel_states_path': channel_states_path,
+            'barcode_timestamps_path': timestamp_path,
             'mappable_timestamp_files': timestamp_files
         }
 

src/openscopenwb/utils/dandi_functions.py

@@ -0,0 +1,118 @@
+import os
+import re
+
+import fsspec
+import h5py
+import pandas as pd
+import json
+import shutil
+import time
+
+
+from pathlib import Path
+from dandi import dandiapi
+from fsspec.implementations.cached import CachingFileSystem
+from pynwb import NWBHDF5IO
+from pynwb.base import Images
+from glob import glob
+from os.path import join
+from shutil import rmtree
+
+from openscopenwb.utils import clean_up_functions as cuf
+from openscopenwb.utils import firebase_functions as fb
+
+
+from dandi.dandiapi import DandiAPIClient as dandi
+def check_sess_info(project, dandi_id, path):
+    """Update a session's dandi location
+
+    Parameters
+    ----------
+    project : str
+        The project id
+    dandi_id : str
+        The dandi id
+    path : str
+        The path to the file
+
+    Returns
+    -------
+    """
+    if "probe" in path:
+        return None
+    sess_id = ""
+    sub_id = ""
+    print(str(path))
+    match = re.search(r'ses-(\d+)', str(path))
+    if match:
+        sess_id = match.group(1)
+        print("Session ID: " + sess_id)
+    else:
+        print("Session ID not found.")
+    if sess_id == "":
+        return None
+    if project !='OpenScopeDendriteCoupling':
+        fb.update_session_dandi(project, sess_id, str(path))
+    else:
+        match = re.search(r'sub-(\d+)', str(path))
+        if match:
+            sub_id = match.group(1)
+            print(sub_id)
+            fb.update_session_dandi(project, sess_id, r'https://dandiarchive.org/dandiset/000336/draft/files?location=' + "sub-" + sub_id ) 
+        return None
+
+def find_dandiset_sessions(project, dandi_id):
+    """Update a project's dandi locations
+
+    Parameters
+    ----------
+    project : str
+        The project id
+    dandi_id : str
+        The dandi id
+
+    Returns
+    -------
+    """
+    os.environ['DANDI_API_KEY'] = cuf.get_creds()
+    dandi_api_key = os.environ['DANDI_API_KEY']
+    my_dandiset = dandiapi.DandiAPIClient(
+        token=dandi_api_key).get_dandiset(dandi_id)
+    for file in my_dandiset.get_assets():
+        path = file.path
+        check_sess_info(project, dandi_id, path)
+
+def generate_exp_list(project, dandi_id):
+    os.environ['DANDI_API_KEY'] = cuf.get_creds()
+    dandi_api_key = os.environ['DANDI_API_KEY']    
+    my_dandiset = dandiapi.DandiAPIClient(token=dandi_api_key).get_dandiset(dandi_id)
+
+    session_experiments = {}  # Dictionary to store experiments per session
+
+    for file in my_dandiset.get_assets():
+        path = file.path
+        match = re.search(r'ses-(\d+)', str(path))
+        if match:
+            sess_id = match.group(1)
+        else:
+            print("ses_id not found for: " + str(path))
+            continue
+        match = re.search(r'acq-(\d+)', str(path))
+        if match:
+            exp_id = match.group(1)
+        else:
+            print("exp_id not found for: " + str(path))
+            continue
+
+        if sess_id not in session_experiments:
+            session_experiments[sess_id] = []  # Create an empty list for a new session
+        if exp_id not in session_experiments[sess_id]:  # Check if the experiment ID is not already present
+            session_experiments[sess_id].append(exp_id) 
+    for i in session_experiments:
+        session_experiments[i].append(len(session_experiments[i]))
+    return session_experiments
+
+
+def save_as_json(session_experiments, filename):
+    with open(filename, 'w') as file:
+        json.dump(session_experiments, file)