diff --git a/allensdk/brain_observatory/behavior/behavior_ophys_api/behavior_ophys_nwb_api.py b/allensdk/brain_observatory/behavior/behavior_ophys_api/behavior_ophys_nwb_api.py index a27e909ae..d8218f793 100644 --- a/allensdk/brain_observatory/behavior/behavior_ophys_api/behavior_ophys_nwb_api.py +++ b/allensdk/brain_observatory/behavior/behavior_ophys_api/behavior_ophys_nwb_api.py @@ -1,27 +1,33 @@ import datetime -from pynwb import NWBFile, NWBHDF5IO -import pandas as pd -import allensdk.brain_observatory.nwb as nwb -import numpy as np -import SimpleITK as sitk -import pytz -import uuid -from pandas.util.testing import assert_frame_equal -import os import math +import uuid +import warnings + import numpy as np -import xarray as xr import pandas as pd +import pytz +import SimpleITK as sitk +import xarray as xr +from pandas.util.testing import assert_frame_equal +from pynwb import NWBHDF5IO, NWBFile -from allensdk.core.lazy_property import LazyProperty +import allensdk.brain_observatory.nwb as nwb +from allensdk.brain_observatory.behavior.metadata_processing import ( + get_expt_description +) +from allensdk.brain_observatory.behavior.behavior_ophys_api import ( + BehaviorOphysApiBase +) +from allensdk.brain_observatory.behavior.schemas import ( + BehaviorTaskParametersSchema, OphysBehaviorMetadataSchema) +from allensdk.brain_observatory.behavior.trials_processing import ( + TRIAL_COLUMN_DESCRIPTION_DICT +) +from allensdk.brain_observatory.nwb.metadata import load_pynwb_extension from allensdk.brain_observatory.nwb.nwb_api import NwbApi from allensdk.brain_observatory.nwb.nwb_utils import set_omitted_stop_time -from allensdk.brain_observatory.behavior.trials_processing import TRIAL_COLUMN_DESCRIPTION_DICT -from allensdk.brain_observatory.behavior.schemas import OphysBehaviorMetadataSchema, BehaviorTaskParametersSchema -from allensdk.brain_observatory.nwb.metadata import load_pynwb_extension -from allensdk.brain_observatory.behavior.behavior_ophys_api import BehaviorOphysApiBase - +from allensdk.core.lazy_property import LazyProperty load_pynwb_extension(OphysBehaviorMetadataSchema, 'ndx-aibs-behavior-ophys') load_pynwb_extension(BehaviorTaskParametersSchema, 'ndx-aibs-behavior-ophys') @@ -29,19 +35,23 @@ class BehaviorOphysNwbApi(NwbApi, BehaviorOphysApiBase): - def __init__(self, *args, **kwargs): self.filter_invalid_rois = kwargs.pop("filter_invalid_rois", False) super(BehaviorOphysNwbApi, self).__init__(*args, **kwargs) - def save(self, session_object): + session_type = str(session_object.metadata['session_type']) + nwbfile = NWBFile( - session_description=str(session_object.metadata['session_type']), + session_description=session_type, identifier=str(session_object.ophys_experiment_id), session_start_time=session_object.metadata['experiment_datetime'], - file_create_date=pytz.utc.localize(datetime.datetime.now()) + file_create_date=pytz.utc.localize(datetime.datetime.now()), + institution="Allen Institute for Brain Science", + keywords=["2-photon", "calcium imaging", "visual cortex", + "behavior", "task"], + experiment_description=get_expt_description(session_type) ) # Add stimulus_timestamps to NWB in-memory object: @@ -93,7 +103,9 @@ def save(self, session_object): nwb.add_task_parameters(nwbfile, session_object.task_parameters) # Add roi metrics to NWB in-memory object: - nwb.add_cell_specimen_table(nwbfile, session_object.cell_specimen_table) + nwb.add_cell_specimen_table(nwbfile, + session_object.cell_specimen_table, + session_object.metadata) # Add dff to NWB in-memory object: nwb.add_dff_traces(nwbfile, session_object.dff_traces, session_object.ophys_timestamps) @@ -122,8 +134,8 @@ def get_running_data_df(self, **kwargs): running_data_df[key] = self.nwbfile.get_acquisition(key).data for key in ['dx']: - if ('running' in self.nwbfile.modules) and (key in self.nwbfile.modules['running'].fields['data_interfaces']): - running_data_df[key] = self.nwbfile.modules['running'].get_data_interface(key).data + if ('running' in self.nwbfile.processing) and (key in self.nwbfile.processing['running'].fields['data_interfaces']): + running_data_df[key] = self.nwbfile.processing['running'].get_data_interface(key).data return running_data_df[['speed', 'dx', 'v_sig', 'v_in']] @@ -131,10 +143,10 @@ def get_stimulus_templates(self, **kwargs): return {key: val.data[:] for key, val in self.nwbfile.stimulus_template.items()} def get_ophys_timestamps(self) -> np.ndarray: - return self.nwbfile.modules['two_photon_imaging'].get_data_interface('dff').roi_response_series['traces'].timestamps[:] + return self.nwbfile.processing['ophys'].get_data_interface('dff').roi_response_series['traces'].timestamps[:] def get_stimulus_timestamps(self) -> np.ndarray: - return self.nwbfile.modules['stimulus'].get_data_interface('timestamps').timestamps[:] + return self.nwbfile.processing['stimulus'].get_data_interface('timestamps').timestamps[:] def get_trials(self) -> pd.DataFrame: trials = self.nwbfile.trials.to_dataframe() @@ -144,35 +156,36 @@ def get_trials(self) -> pd.DataFrame: return trials def get_licks(self) -> np.ndarray: - if 'licking' in self.nwbfile.modules: - return pd.DataFrame({'time': self.nwbfile.modules['licking'].get_data_interface('licks')['timestamps'].timestamps[:]}) + if 'licking' in self.nwbfile.processing: + return pd.DataFrame({'time': self.nwbfile.processing['licking'].get_data_interface('licks')['timestamps'].timestamps[:]}) else: return pd.DataFrame({'time': []}) def get_rewards(self) -> np.ndarray: - if 'rewards' in self.nwbfile.modules: - time = self.nwbfile.modules['rewards'].get_data_interface('autorewarded').timestamps[:] - autorewarded = self.nwbfile.modules['rewards'].get_data_interface('autorewarded').data[:] - volume = self.nwbfile.modules['rewards'].get_data_interface('volume').data[:] + if 'rewards' in self.nwbfile.processing: + time = self.nwbfile.processing['rewards'].get_data_interface('autorewarded').timestamps[:] + autorewarded = self.nwbfile.processing['rewards'].get_data_interface('autorewarded').data[:] + volume = self.nwbfile.processing['rewards'].get_data_interface('volume').data[:] return pd.DataFrame({'volume': volume, 'timestamps': time, 'autorewarded': autorewarded}).set_index('timestamps') else: return pd.DataFrame({'volume': [], 'timestamps': [], 'autorewarded': []}).set_index('timestamps') def get_max_projection(self, image_api=None) -> sitk.Image: - return self.get_image('max_projection', 'two_photon_imaging', image_api=image_api) + return self.get_image('max_projection', 'ophys', image_api=image_api) def get_average_projection(self, image_api=None) -> sitk.Image: - return self.get_image('average_image', 'two_photon_imaging', image_api=image_api) + return self.get_image('average_image', 'ophys', image_api=image_api) def get_segmentation_mask_image(self, image_api=None) -> sitk.Image: - return self.get_image('segmentation_mask_image', 'two_photon_imaging', image_api=image_api) + return self.get_image('segmentation_mask_image', 'ophys', image_api=image_api) def get_metadata(self) -> dict: metadata_nwb_obj = self.nwbfile.lab_meta_data['metadata'] - data = OphysBehaviorMetadataSchema(exclude=['experiment_datetime']).dump(metadata_nwb_obj) + data = OphysBehaviorMetadataSchema( + exclude=['experiment_datetime']).dump(metadata_nwb_obj) - # Add subject related metadata to behavior ophys metadata + # Add pyNWB Subject metadata to behavior ophys session metadata nwb_subject = self.nwbfile.subject data['LabTracks_ID'] = int(nwb_subject.subject_id) data['sex'] = nwb_subject.sex @@ -181,9 +194,31 @@ def get_metadata(self) -> dict: data['reporter_line'] = list(nwb_subject.reporter_line) data['driver_line'] = list(nwb_subject.driver_line) - experiment_datetime = metadata_nwb_obj.experiment_datetime - data['experiment_datetime'] = OphysBehaviorMetadataSchema().load({'experiment_datetime': experiment_datetime}, partial=True)['experiment_datetime'] - data['behavior_session_uuid'] = uuid.UUID(data['behavior_session_uuid']) + # Add pyNWB OpticalChannel and ImagingPlane metadata to behavior ophys + # session metadata + try: + ophys_module = self.nwbfile.processing['ophys'] + except KeyError: + warnings.warn("Could not locate 'ophys' module in " + "NWB file. The following metadata fields will be " + "missing: 'ophys_frame_rate', 'indicator', " + "'targeted_structure', 'excitation_lambda', " + "'emission_lambda'") + else: + image_seg = ophys_module.data_interfaces['image_segmentation'] + imaging_plane = image_seg.plane_segmentations['cell_specimen_table'].imaging_plane + optical_channel = imaging_plane.optical_channel[0] + + data['ophys_frame_rate'] = imaging_plane.imaging_rate + data['indicator'] = imaging_plane.indicator + data['targeted_structure'] = imaging_plane.location + data['excitation_lambda'] = imaging_plane.excitation_lambda + data['emission_lambda'] = optical_channel.emission_lambda + + # Add other metadata stored in nwb file to behavior ophys session meta + data['experiment_datetime'] = self.nwbfile.session_start_time + data['behavior_session_uuid'] = uuid.UUID( + data['behavior_session_uuid']) return data def get_task_parameters(self) -> dict: @@ -193,10 +228,11 @@ def get_task_parameters(self) -> dict: return data def get_cell_specimen_table(self) -> pd.DataFrame: - df = self.nwbfile.modules['two_photon_imaging'].data_interfaces['image_segmentation'].plane_segmentations['cell_specimen_table'].to_dataframe() + # NOTE: ROI masks are stored in full frame width and height arrays + df = self.nwbfile.processing['ophys'].data_interfaces['image_segmentation'].plane_segmentations['cell_specimen_table'].to_dataframe() df.index.rename('cell_roi_id', inplace=True) df['cell_specimen_id'] = [None if csid == -1 else csid for csid in df['cell_specimen_id'].values] - df['image_mask'] = [mask.astype(bool) for mask in df['image_mask'].values] + df.reset_index(inplace=True) df.set_index('cell_specimen_id', inplace=True) @@ -206,31 +242,40 @@ def get_cell_specimen_table(self) -> pd.DataFrame: return df def get_dff_traces(self) -> pd.DataFrame: - dff_nwb = self.nwbfile.modules['two_photon_imaging'].data_interfaces['dff'].roi_response_series['traces'] - dff_traces = dff_nwb.data[:] + dff_nwb = self.nwbfile.processing['ophys'].data_interfaces['dff'].roi_response_series['traces'] + # dff traces stored as timepoints x rois in NWB + # We want rois x timepoints, hence the transpose + dff_traces = dff_nwb.data[:].T number_of_cells, number_of_dff_frames = dff_traces.shape num_of_timestamps = len(self.get_ophys_timestamps()) assert num_of_timestamps == number_of_dff_frames - - df = pd.DataFrame({'dff': [x for x in dff_traces]}, index=pd.Index(data=dff_nwb.rois.table.id[:], name='cell_roi_id')) + + df = pd.DataFrame({'dff': dff_traces.tolist()}, + index=pd.Index(data=dff_nwb.rois.table.id[:], + name='cell_roi_id')) cell_specimen_table = self.get_cell_specimen_table() df = cell_specimen_table[['cell_roi_id']].join(df, on='cell_roi_id') return df def get_corrected_fluorescence_traces(self) -> pd.DataFrame: - corrected_fluorescence_nwb = self.nwbfile.modules['two_photon_imaging'].data_interfaces['corrected_fluorescence'].roi_response_series['traces'] - df = pd.DataFrame({'corrected_fluorescence': [x for x in corrected_fluorescence_nwb.data[:]]}, - index=pd.Index(data=corrected_fluorescence_nwb.rois.table.id[:], name='cell_roi_id')) + corrected_fluorescence_nwb = self.nwbfile.processing['ophys'].data_interfaces['corrected_fluorescence'].roi_response_series['traces'] + # f traces stored as timepoints x rois in NWB + # We want rois x timepoints, hence the transpose + f_traces = corrected_fluorescence_nwb.data[:].T + df = pd.DataFrame({'corrected_fluorescence': f_traces.tolist()}, + index=pd.Index(data=corrected_fluorescence_nwb.rois.table.id[:], + name='cell_roi_id')) cell_specimen_table = self.get_cell_specimen_table() df = cell_specimen_table[['cell_roi_id']].join(df, on='cell_roi_id') return df def get_motion_correction(self) -> pd.DataFrame: + ophys_module = self.nwbfile.processing['ophys'] motion_correction_data = {} - motion_correction_data['x'] = self.nwbfile.modules['motion_correction'].get_data_interface('x').data[:] - motion_correction_data['y'] = self.nwbfile.modules['motion_correction'].get_data_interface('y').data[:] + motion_correction_data['x'] = ophys_module.get_data_interface('ophys_motion_correction_x').data[:] + motion_correction_data['y'] = ophys_module.get_data_interface('ophys_motion_correction_y').data[:] return pd.DataFrame(motion_correction_data) @@ -296,4 +341,4 @@ def compare_fields(x1, x2, err_msg=""): assert x1[key] == x2[key], key_err_msg else: - assert x1 == x2, err_msg \ No newline at end of file + assert x1 == x2, err_msg diff --git a/allensdk/brain_observatory/behavior/metadata_processing.py b/allensdk/brain_observatory/behavior/metadata_processing.py index 838554085..dd873666a 100644 --- a/allensdk/brain_observatory/behavior/metadata_processing.py +++ b/allensdk/brain_observatory/behavior/metadata_processing.py @@ -1,3 +1,77 @@ +OPHYS_1_3_DESCRIPTION = ( + "2-photon calcium imaging in the visual cortex of the mouse " + "brain as the mouse performs a visual change detection task " + "with a set of natural scenes upon which it has previously been " + "trained." +) +OPHYS_2_DESCRIPTION = ( + "2-photon calcium imaging in the visual cortex of the " + "mouse brain as the mouse is shown images from a " + "change detection task with a set of natural scenes " + "upon which it has previously been trained, but with " + "the lick-response sensor withdrawn (passive/open " + "loop mode)." +) +OPHYS_4_6_DESCRIPTION = ( + "2-photon calcium imaging in the visual cortex of the mouse " + "brain as the mouse performs a visual change detection task " + "with a set of natural scenes that are unique from those on " + "which it was previously trained." +) +OPHYS_5_DESCRIPTION = ( + "2-photon calcium imaging in the visual cortex of the " + "mouse brain as the mouse is shown images from a " + "change detection task with a set of natural scenes " + "that are unique from those on which it was " + "previously trained, but with the lick-response " + "sensor withdrawn (passive/open loop mode)." +) + + +def get_expt_description(session_type: str) -> str: + """Determine a behavior ophys session's experiment description based on + session type. + + Parameters + ---------- + session_type : str + A session description string (e.g. OPHYS_1_images_B ) + + Returns + ------- + str + A description of the experiment based on the session_type. + + Raises + ------ + RuntimeError + Behavior ophys sessions should only have 6 different session types. + Unknown session types (or malformed session_type strings) will raise + an error. + """ + # Experiment descriptions for different session types: + # OPHYS_1 -> OPHYS_6 + ophys_1_3 = dict.fromkeys(["OPHYS_1", "OPHYS_3"], OPHYS_1_3_DESCRIPTION) + ophys_4_6 = dict.fromkeys(["OPHYS_4", "OPHYS_6"], OPHYS_4_6_DESCRIPTION) + ophys_2_5 = {"OPHYS_2": OPHYS_2_DESCRIPTION, + "OPHYS_5": OPHYS_5_DESCRIPTION} + + expt_description_dict = {**ophys_1_3, **ophys_2_5, **ophys_4_6} + + # Session type string will look something like: OPHYS_4_images_A + truncated_session_type = "_".join(session_type.split("_")[:2]) + + try: + return expt_description_dict[truncated_session_type] + except KeyError as e: + e_msg = ( + f"Encountered an unknown session type " + f"({truncated_session_type}) when trying to determine " + f"experiment descriptions. Valid session types are: " + f"{expt_description_dict.keys()}") + raise RuntimeError(e_msg) from e + + def get_task_parameters(data): task_parameters = {} @@ -15,4 +89,4 @@ def get_task_parameters(data): n_stimulus_frames += sum(stim_table.get("draw_log", [])) task_parameters['n_stimulus_frames'] = n_stimulus_frames - return task_parameters \ No newline at end of file + return task_parameters diff --git a/allensdk/brain_observatory/behavior/schemas.py b/allensdk/brain_observatory/behavior/schemas.py index a57e92aa7..e9364a8b1 100644 --- a/allensdk/brain_observatory/behavior/schemas.py +++ b/allensdk/brain_observatory/behavior/schemas.py @@ -22,6 +22,9 @@ class SubjectMetadataSchema(RaisingSchema): neurodata_type = 'BehaviorSubject' neurodata_type_inc = 'Subject' neurodata_doc = "Metadata for an AIBS behavior or behavior + ophys subject" + # Fields to skip converting to extension + # In this case they already exist in the 'Subject' builtin pyNWB class + neurodata_skip = {"age", "genotype", "sex", "subject_id"} age = fields.String( doc='Age of the specimen donor/subject', @@ -70,17 +73,43 @@ class BehaviorMetadataSchema(RaisingSchema): ) -class OphysMetadataSchema(RaisingSchema): - """This schema contains metadata pertaining to optical physiology (ophys). - """ +class NwbOphysMetadataSchema(RaisingSchema): + """This schema contains fields that will be stored in pyNWB base classes + pertaining to optical physiology.""" + # 'emission_lambda' will be stored in + # pyNWB OpticalChannel 'emission_lambda' attr emission_lambda = fields.Float( - doc='emission_lambda', + doc='Emission lambda of fluorescent indicator', required=True, ) + # 'excitation_lambda' will be stored in the pyNWB ImagingPlane + # 'excitation_lambda' attr excitation_lambda = fields.Float( - doc='excitation_lambda', + doc='Excitation lambda of fluorescent indicator', + required=True, + ) + # 'indicator' will be stored in the pyNWB ImagingPlane 'indicator' attr + indicator = fields.String( + doc='Name of optical physiology fluorescent indicator', + required=True, + ) + # 'targeted_structure' will be stored in the pyNWB + # ImagingPlane 'location' attr + targeted_structure = fields.String( + doc='Anatomical structure targeted for two-photon acquisition', required=True, ) + # 'ophys_frame_rate' will be stored in the pyNWB ImagingPlane + # 'imaging_rate' attr + ophys_frame_rate = fields.Float( + doc='Frame rate (frames/second) of the two-photon microscope', + required=True, + ) + + +class OphysMetadataSchema(NwbOphysMetadataSchema): + """This schema contains metadata pertaining to optical physiology (ophys). + """ experiment_container_id = fields.Int( doc='Container ID for the container that contains this ophys session', required=True, @@ -90,24 +119,20 @@ class OphysMetadataSchema(RaisingSchema): 'targeted for two-photon acquisition'), required=True, ) - indicator = fields.String( - doc='indicator', - required=True, - ) ophys_experiment_id = fields.Int( doc='Id for this ophys session', required=True, ) - ophys_frame_rate = fields.Float( - doc='Frame rate (frames/second) of the two-photon microscope', + rig_name = fields.String( + doc='Name of optical physiology experiment rig', required=True, ) - rig_name = fields.String( - doc='name of two-photon rig', + field_of_view_width = fields.Int( + doc='Width of optical physiology imaging plane in pixels', required=True, ) - targeted_structure = fields.String( - doc='Anatomical structure targeted for two-photon acquisition', + field_of_view_height = fields.Int( + doc='Height of optical physiology imaging plane in pixels', required=True, ) @@ -120,24 +145,24 @@ class OphysBehaviorMetadataSchema(BehaviorMetadataSchema, OphysMetadataSchema): neurodata_type = 'OphysBehaviorMetadata' neurodata_type_inc = 'LabMetaData' neurodata_doc = "Metadata for behavior + ophys experiments" + # Fields to skip converting to extension + # They already exist as attributes for the following pyNWB classes: + # OpticalChannel, ImagingPlane, NWBFile + neurodata_skip = {"emission_lambda", "excitation_lambda", "indicator", + "targeted_structure", "experiment_datetime", + "ophys_frame_rate"} session_type = fields.String( doc='Experimental session description', allow_none=True, required=True, ) + # 'experiment_datetime' will be stored in + # pynwb NWBFile 'session_start_time' attr experiment_datetime = fields.DateTime( doc='Date of the experiment (UTC, as string)', required=True, ) - field_of_view_width = fields.Int( - doc='field_of_view_width', - required=True, - ) - field_of_view_height = fields.Int( - doc='field_of_view_height', - required=True, - ) class CompleteOphysBehaviorMetadataSchema(OphysBehaviorMetadataSchema, @@ -159,46 +184,49 @@ class BehaviorTaskParametersSchema(RaisingSchema): blank_duration_sec = fields.List( fields.Float, - doc='blank duration in seconds', + doc=('The lower and upper bound (in seconds) for a randomly chosen ' + 'inter-stimulus interval duration for a trial'), required=True, shape=(2,), ) stimulus_duration_sec = fields.Float( - doc='duration of each stimulus presentation in seconds', + doc='Duration of each stimulus presentation in seconds', required=True, ) omitted_flash_fraction = fields.Float( - doc='omitted_flash_fraction', + doc='Fraction of flashes/image presentations that were omitted', required=True, allow_nan=True, ) response_window_sec = fields.List( fields.Float, - doc='response_window in seconds', + doc=('The lower and upper bound (in seconds) for a randomly chosen ' + 'time window where subject response influences trial outcome'), required=True, shape=(2,), ) reward_volume = fields.Float( - doc='reward_volume', + doc='Volume of water (in mL) delivered as reward', required=True, ) stage = fields.String( - doc='stage', + doc='Stage of behavioral task', required=True, ) stimulus = fields.String( - doc='stimulus', + doc='Stimulus type', required=True, ) stimulus_distribution = fields.String( - doc='stimulus_distribution', + doc=("Distribution type of drawing change times " + "(e.g. 'geometric', 'exponential')"), required=True, ) task = fields.String( - doc='task', + doc='The name of the behavioral task', required=True, ) n_stimulus_frames = fields.Int( - doc='n_stimulus_frames', + doc='Total number of stimuli frames', required=True, ) diff --git a/allensdk/brain_observatory/behavior/write_nwb/_schemas.py b/allensdk/brain_observatory/behavior/write_nwb/_schemas.py index 26a1259ad..38f6df620 100644 --- a/allensdk/brain_observatory/behavior/write_nwb/_schemas.py +++ b/allensdk/brain_observatory/behavior/write_nwb/_schemas.py @@ -8,8 +8,8 @@ class CellSpecimenTable(RaisingSchema): cell_roi_id = Dict(String, Int, required=True) cell_specimen_id = Dict(String, Int(allow_none=True), required=True) - x = Dict(String, Float, required=True) - y = Dict(String, Float, required=True) + x = Dict(String, Int, required=True) + y = Dict(String, Int, required=True) max_correction_up = Dict(String, Float, required=True) max_correction_right = Dict(String, Float, required=True) max_correction_down = Dict(String, Float, required=True) diff --git a/allensdk/brain_observatory/nwb/__init__.py b/allensdk/brain_observatory/nwb/__init__.py index 727508100..20e5976c3 100644 --- a/allensdk/brain_observatory/nwb/__init__.py +++ b/allensdk/brain_observatory/nwb/__init__.py @@ -1,9 +1,10 @@ import logging import warnings from pathlib import Path -from typing import Iterable, Tuple +from typing import Iterable import h5py +import marshmallow import numpy as np import pandas as pd import datetime @@ -23,8 +24,9 @@ from allensdk.brain_observatory.behavior.image_api import Image from allensdk.brain_observatory.behavior.image_api import ImageApi from allensdk.brain_observatory.behavior.schemas import ( - CompleteOphysBehaviorMetadataSchema, OphysBehaviorMetadataSchema, - BehaviorTaskParametersSchema, SubjectMetadataSchema + CompleteOphysBehaviorMetadataSchema, NwbOphysMetadataSchema, + OphysBehaviorMetadataSchema, BehaviorTaskParametersSchema, + SubjectMetadataSchema ) from allensdk.brain_observatory.nwb.metadata import load_pynwb_extension @@ -363,7 +365,7 @@ def add_running_data_df_to_nwbfile(nwbfile, running_data_df, unit_dict, index_ke add_running_speed_to_nwbfile(nwbfile, running_speed, name='speed', unit=unit_dict['speed']) - running_mod = nwbfile.modules['running'] + running_mod = nwbfile.processing['running'] timestamps_ts = running_mod.get_data_interface('speed').timestamps running_dx_series = TimeSeries( @@ -473,7 +475,7 @@ def add_stimulus_presentations(nwbfile, stimulus_table, tag='stimulus_time_inter """ stimulus_table = stimulus_table.copy() - ts = nwbfile.modules['stimulus'].get_data_interface('timestamps') + ts = nwbfile.processing['stimulus'].get_data_interface('timestamps') possible_names = {'stimulus_name', 'image_name'} stimulus_name_column = get_column_name(stimulus_table.columns, possible_names) @@ -650,14 +652,14 @@ def add_rewards(nwbfile, rewards_df): name='volume', data=rewards_df.volume.values, timestamps=rewards_df.index.values, - unit='ml' + unit='mL' ) autorewarded_ts = TimeSeries( name='autorewarded', data=rewards_df.autorewarded.values, timestamps=reward_volume_ts.timestamps, - unit=None + unit='mL' ) rewards_mod = ProcessingModule('rewards', 'Licking behavior processing module') @@ -686,11 +688,11 @@ def add_image(nwbfile, image_data, image_name, module_name, module_description, assert spacing[0] == spacing[1] and len(spacing) == 2 and unit == 'mm' - if module_name not in nwbfile.modules: + if module_name not in nwbfile.processing: ophys_mod = ProcessingModule(module_name, module_description) nwbfile.add_processing_module(ophys_mod) else: - ophys_mod = nwbfile.modules[module_name] + ophys_mod = nwbfile.processing[module_name] image = GrayscaleImage(image_name, data, resolution=spacing[0] / 10, description=description) @@ -706,17 +708,17 @@ def add_image(nwbfile, image_data, image_name, module_name, module_description, def add_max_projection(nwbfile, max_projection, image_api=None): - add_image(nwbfile, max_projection, 'max_projection', 'two_photon_imaging', 'Ophys timestamps processing module', image_api=image_api) + add_image(nwbfile, max_projection, 'max_projection', 'ophys', 'Ophys processing module', image_api=image_api) def add_average_image(nwbfile, average_image, image_api=None): - add_image(nwbfile, average_image, 'average_image', 'two_photon_imaging', 'Ophys timestamps processing module', image_api=image_api) + add_image(nwbfile, average_image, 'average_image', 'ophys', 'Ophys processing module', image_api=image_api) def add_segmentation_mask_image(nwbfile, segmentation_mask_image, image_api=None): - add_image(nwbfile, segmentation_mask_image, 'segmentation_mask_image', 'two_photon_imaging', 'Ophys timestamps processing module', image_api=image_api) + add_image(nwbfile, segmentation_mask_image, 'segmentation_mask_image', 'ophys', 'Ophys processing module', image_api=image_api) def add_stimulus_index(nwbfile, stimulus_index, nwb_template): @@ -756,10 +758,15 @@ def add_metadata(nwbfile, metadata: dict): genotype=subject_metadata["genotype"], subject_id=str(subject_metadata["subject_id"]), reporter_line=subject_metadata["reporter_line"], - sex=subject_metadata["sex"]) + sex=subject_metadata["sex"], + species='Mus musculus') nwbfile.subject = nwb_subject - # Rest of metadata can go into our custom extension + # Remove metadata that will go into pyNWB base classes + for key in OphysBehaviorMetadataSchema.neurodata_skip: + metadata_clean.pop(key, None) + + # Remaining metadata can go into our custom extension new_metadata_dict = {} for key, val in metadata_clean.items(): if isinstance(val, list): @@ -795,11 +802,13 @@ def add_task_parameters(nwbfile, task_parameters): def add_cell_specimen_table(nwbfile: NWBFile, - cell_specimen_table: pd.DataFrame): + cell_specimen_table: pd.DataFrame, + session_metadata: dict): """ This function takes the cell specimen table and writes the ROIs contained within. It writes these to a new NWB imaging plane based off the previously supplied metadata + Parameters ---------- nwbfile: NWBFile @@ -810,38 +819,40 @@ def add_cell_specimen_table(nwbfile: NWBFile, experiment, stored in json file and loaded. example: /home/nicholasc/projects/allensdk/allensdk/test/ brain_observatory/behavior/cell_specimen_table_789359614.json + session_metadata: dict + Dictionary containing cell_specimen_table related metadata. Should + include at minimum the following fields: + "emission_lambda", "excitation_lambda", "indicator", + "targeted_structure", and ophys_frame_rate" Returns ------- nwbfile: NWBFile The altered in memory NWBFile object that now has a specimen table """ + cell_specimen_metadata = NwbOphysMetadataSchema().load( + session_metadata, unknown=marshmallow.EXCLUDE) cell_roi_table = cell_specimen_table.reset_index().set_index('cell_roi_id') # Device: device_name = nwbfile.lab_meta_data['metadata'].rig_name nwbfile.create_device(device_name, - "Allen Brain Observatory") + "Allen Brain Observatory - Scientifica 2P Rig") device = nwbfile.get_device(device_name) - # Location: - location_description = "Area: {}, Depth: {} um".format( - nwbfile.lab_meta_data['metadata'].targeted_structure, - nwbfile.lab_meta_data['metadata'].imaging_depth) - # FOV: fov_width = nwbfile.lab_meta_data['metadata'].field_of_view_width fov_height = nwbfile.lab_meta_data['metadata'].field_of_view_height imaging_plane_description = "{} field of view in {} at depth {} um".format( (fov_width, fov_height), - nwbfile.lab_meta_data['metadata'].targeted_structure, + cell_specimen_metadata['targeted_structure'], nwbfile.lab_meta_data['metadata'].imaging_depth) # Optical Channel: optical_channel = OpticalChannel( name='channel_1', description='2P Optical Channel', - emission_lambda=nwbfile.lab_meta_data['metadata'].emission_lambda) + emission_lambda=cell_specimen_metadata['emission_lambda']) # Imaging Plane: imaging_plane = nwbfile.create_imaging_plane( @@ -849,25 +860,21 @@ def add_cell_specimen_table(nwbfile: NWBFile, optical_channel=optical_channel, description=imaging_plane_description, device=device, - excitation_lambda=nwbfile.lab_meta_data['metadata'].excitation_lambda, - imaging_rate=nwbfile.lab_meta_data['metadata'].ophys_frame_rate, - indicator=nwbfile.lab_meta_data['metadata'].indicator, - location=location_description, - manifold=[], # Should this be passed in for future support? - conversion=1.0, - unit='unknown', # Should this be passed in for future support? - reference_frame='unknown') # Should this be passed in for future support? + excitation_lambda=cell_specimen_metadata['excitation_lambda'], + imaging_rate=cell_specimen_metadata['ophys_frame_rate'], + indicator=cell_specimen_metadata['indicator'], + location=cell_specimen_metadata['targeted_structure']) # Image Segmentation: image_segmentation = ImageSegmentation(name="image_segmentation") - if 'two_photon_imaging' not in nwbfile.modules: - two_photon_imaging_module = ProcessingModule('two_photon_imaging', '2P processing module') - nwbfile.add_processing_module(two_photon_imaging_module) + if 'ophys' not in nwbfile.processing: + ophys_module = ProcessingModule('ophys', 'Ophys processing module') + nwbfile.add_processing_module(ophys_module) else: - two_photon_imaging_module = nwbfile.modules['two_photon_imaging'] + ophys_module = nwbfile.processing['ophys'] - two_photon_imaging_module.add_data_interface(image_segmentation) + ophys_module.add_data_interface(image_segmentation) # Plane Segmentation: plane_segmentation = image_segmentation.create_plane_segmentation( @@ -886,15 +893,19 @@ def add_cell_specimen_table(nwbfile: NWBFile, "No Description Available")) # go through each roi and add it to the plan segmentation object - for cell_roi_id, row in cell_roi_table.iterrows(): - sub_mask = np.array(row.pop('image_mask')) - curr_roi = roi.create_roi_mask(fov_width, fov_height, [(fov_width - 1), 0, (fov_height - 1), 0], - roi_mask=sub_mask) - mask = curr_roi.get_mask_plane() - csid = row.pop('cell_specimen_id') - row['cell_specimen_id'] = -1 if csid is None else csid - row['id'] = cell_roi_id - plane_segmentation.add_roi(image_mask=mask, **row.to_dict()) + for cell_roi_id, table_row in cell_roi_table.iterrows(): + + # NOTE: The 'image_mask' in this cell_roi_table has already been + # processing by the allensdk.internal.api.ophys_lims_api + # get_cell_specimen_table() method. As a result, the ROI is stored in + # an array that is the same shape as the FULL field of view of the + # experiment (e.g. 512 x 512). + mask = table_row.pop('image_mask') + + csid = table_row.pop('cell_specimen_id') + table_row['cell_specimen_id'] = -1 if csid is None else csid + table_row['id'] = cell_roi_id + plane_segmentation.add_roi(image_mask=mask, **table_row.to_dict()) return nwbfile @@ -902,11 +913,12 @@ def add_cell_specimen_table(nwbfile: NWBFile, def add_dff_traces(nwbfile, dff_traces, ophys_timestamps): dff_traces = dff_traces.reset_index().set_index('cell_roi_id')[['dff']] - twop_module = nwbfile.modules['two_photon_imaging'] - data = np.array([dff_traces.loc[cell_roi_id].dff for cell_roi_id in dff_traces.index.values]) - # assert len(ophys_timestamps.timestamps) == len(data) + ophys_module = nwbfile.processing['ophys'] + # trace data in the form of rois x timepoints + trace_data = np.array([dff_traces.loc[cell_roi_id].dff + for cell_roi_id in dff_traces.index.values]) - cell_specimen_table = nwbfile.modules['two_photon_imaging'].data_interfaces['image_segmentation'].plane_segmentations['cell_specimen_table'] + cell_specimen_table = nwbfile.processing['ophys'].data_interfaces['image_segmentation'].plane_segmentations['cell_specimen_table'] roi_table_region = cell_specimen_table.create_roi_table_region( description="segmented cells labeled by cell_specimen_id", region=slice(len(dff_traces))) @@ -914,11 +926,11 @@ def add_dff_traces(nwbfile, dff_traces, ophys_timestamps): # Create/Add dff modules and interfaces: assert dff_traces.index.name == 'cell_roi_id' dff_interface = DfOverF(name='dff') - twop_module.add_data_interface(dff_interface) + ophys_module.add_data_interface(dff_interface) dff_interface.create_roi_response_series( name='traces', - data=data, + data=trace_data.T, # Should be stored as timepoints x rois unit='NA', rois=roi_table_region, timestamps=ophys_timestamps) @@ -931,14 +943,19 @@ def add_corrected_fluorescence_traces(nwbfile, corrected_fluorescence_traces): # Create/Add corrected_fluorescence_traces modules and interfaces: assert corrected_fluorescence_traces.index.name == 'cell_roi_id' - twop_module = nwbfile.modules['two_photon_imaging'] - roi_table_region = nwbfile.modules['two_photon_imaging'].data_interfaces['dff'].roi_response_series['traces'].rois - ophys_timestamps = twop_module.get_data_interface('dff').roi_response_series['traces'].timestamps + ophys_module = nwbfile.processing['ophys'] + # trace data in the form of rois x timepoints + f_trace_data = np.array([corrected_fluorescence_traces.loc[cell_roi_id].corrected_fluorescence + for cell_roi_id in corrected_fluorescence_traces.index.values]) + + roi_table_region = nwbfile.processing['ophys'].data_interfaces['dff'].roi_response_series['traces'].rois + ophys_timestamps = ophys_module.get_data_interface('dff').roi_response_series['traces'].timestamps f_interface = Fluorescence(name='corrected_fluorescence') - twop_module.add_data_interface(f_interface) + ophys_module.add_data_interface(f_interface) + f_interface.create_roi_response_series( name='traces', - data=np.array([corrected_fluorescence_traces.loc[cell_roi_id].corrected_fluorescence for cell_roi_id in corrected_fluorescence_traces.index.values]), + data=f_trace_data.T, # Should be stored as timepoints x rois unit='NA', rois=roi_table_region, timestamps=ophys_timestamps) @@ -948,24 +965,22 @@ def add_corrected_fluorescence_traces(nwbfile, corrected_fluorescence_traces): def add_motion_correction(nwbfile, motion_correction): - twop_module = nwbfile.modules['two_photon_imaging'] - ophys_timestamps = twop_module.get_data_interface('dff').roi_response_series['traces'].timestamps + ophys_module = nwbfile.processing['ophys'] + ophys_timestamps = ophys_module.get_data_interface('dff').roi_response_series['traces'].timestamps t1 = TimeSeries( - name='x', + name='ophys_motion_correction_x', data=motion_correction['x'].values, timestamps=ophys_timestamps, unit='pixels' ) t2 = TimeSeries( - name='y', + name='ophys_motion_correction_y', data=motion_correction['y'].values, timestamps=ophys_timestamps, unit='pixels' ) - motion_module = ProcessingModule('motion_correction', 'Motion Correction processing module') - motion_module.add_data_interface(t1) - motion_module.add_data_interface(t2) - nwbfile.add_processing_module(motion_module) + ophys_module.add_data_interface(t1) + ophys_module.add_data_interface(t2) diff --git a/allensdk/brain_observatory/nwb/metadata.py b/allensdk/brain_observatory/nwb/metadata.py index c646238eb..5bd52c5e4 100644 --- a/allensdk/brain_observatory/nwb/metadata.py +++ b/allensdk/brain_observatory/nwb/metadata.py @@ -9,11 +9,20 @@ def extract_from_schema(schema): + if hasattr(schema, 'neurodata_skip'): + fields_to_skip = schema.neurodata_skip + else: + fields_to_skip = set() + # Extract fields from Schema: docval_list = [{'name': 'name', 'type': str, 'doc': 'name'}] attributes = [] nwbfields_list = [] for name, val in schema().fields.items(): + + if name in fields_to_skip: + continue + if type(val) == fields.List: attributes.append(NWBAttributeSpec(name=name, dtype=STYPE_DICT[type(val)], diff --git a/allensdk/brain_observatory/nwb/ndx-aibs-behavior-ophys.extension.yaml b/allensdk/brain_observatory/nwb/ndx-aibs-behavior-ophys.extension.yaml index 976c2fcd6..bc5ef12dd 100644 --- a/allensdk/brain_observatory/nwb/ndx-aibs-behavior-ophys.extension.yaml +++ b/allensdk/brain_observatory/nwb/ndx-aibs-behavior-ophys.extension.yaml @@ -3,112 +3,84 @@ groups: neurodata_type_inc: LabMetaData doc: Metadata for behavior or behavior + ophys task parameters attributes: + - name: task + dtype: text + doc: The name of the behavioral task - name: omitted_flash_fraction dtype: float - doc: omitted_flash_fraction - - name: stimulus_duration_sec - dtype: float - doc: duration of each stimulus presentation in seconds - - name: reward_volume - dtype: float - doc: reward_volume - - name: response_window_sec + doc: Fraction of flashes/image presentations that were omitted + - name: blank_duration_sec dtype: text shape: - 2 - doc: response_window in seconds - - name: blank_duration_sec + doc: The lower and upper bound (in seconds) for a randomly chosen inter-stimulus + interval duration for a trial + - name: n_stimulus_frames + dtype: int + doc: Total number of stimuli frames + - name: stimulus + dtype: text + doc: Stimulus type + - name: response_window_sec dtype: text shape: - 2 - doc: blank duration in seconds - - name: stage - dtype: text - doc: stage + doc: The lower and upper bound (in seconds) for a randomly chosen time window + where subject response influences trial outcome - name: stimulus_distribution dtype: text - doc: stimulus_distribution - - name: n_stimulus_frames - dtype: int - doc: n_stimulus_frames - - name: task - dtype: text - doc: task - - name: stimulus + doc: Distribution type of drawing change times (e.g. 'geometric', 'exponential') + - name: reward_volume + dtype: float + doc: Volume of water (in mL) delivered as reward + - name: stimulus_duration_sec + dtype: float + doc: Duration of each stimulus presentation in seconds + - name: stage dtype: text - doc: stimulus + doc: Stage of behavioral task - neurodata_type_def: BehaviorSubject neurodata_type_inc: Subject doc: Metadata for an AIBS behavior or behavior + ophys subject attributes: - - name: genotype - dtype: text - doc: full genotype of subject - - name: reporter_line - dtype: text - shape: - - null - doc: Reporter line of subject - - name: sex - dtype: text - doc: Sex of the specimen donor/subject - name: driver_line dtype: text shape: - null doc: Driver line of subject - - name: age + - name: reporter_line dtype: text - doc: Age of the specimen donor/subject - - name: subject_id - dtype: int - doc: LabTracks ID of subject + shape: + - null + doc: Reporter line of subject - neurodata_type_def: OphysBehaviorMetadata neurodata_type_inc: LabMetaData doc: Metadata for behavior + ophys experiments attributes: + - name: imaging_depth + dtype: int + doc: Depth (microns) below the cortical surface targeted for two-photon acquisition + - name: session_type + dtype: text + doc: Experimental session description - name: rig_name dtype: text - doc: name of two-photon rig - - name: stimulus_frame_rate - dtype: float - doc: Frame rate (frames/second) of the visual_stimulus from the monitor - - name: emission_lambda - dtype: float - doc: emission_lambda + doc: Name of optical physiology experiment rig + - name: ophys_experiment_id + dtype: int + doc: Id for this ophys session - name: experiment_container_id dtype: int doc: Container ID for the container that contains this ophys session - - name: field_of_view_height - dtype: int - doc: field_of_view_height - - name: excitation_lambda + - name: stimulus_frame_rate dtype: float - doc: excitation_lambda - - name: indicator - dtype: text - doc: indicator + doc: Frame rate (frames/second) of the visual_stimulus from the monitor - name: behavior_session_uuid dtype: text doc: MTrain record for session, also called foraging_id - - name: session_type - dtype: text - doc: Experimental session description - - name: field_of_view_width - dtype: int - doc: field_of_view_width - - name: targeted_structure - dtype: text - doc: Anatomical structure targeted for two-photon acquisition - - name: ophys_experiment_id + - name: field_of_view_height dtype: int - doc: Id for this ophys session - - name: experiment_datetime - dtype: text - doc: Date of the experiment (UTC, as string) - - name: imaging_depth + doc: Height of optical physiology imaging plane in pixels + - name: field_of_view_width dtype: int - doc: Depth (microns) below the cortical surface targeted for two-photon acquisition - - name: ophys_frame_rate - dtype: float - doc: Frame rate (frames/second) of the two-photon microscope + doc: Width of optical physiology imaging plane in pixels diff --git a/allensdk/internal/api/ophys_lims_api.py b/allensdk/internal/api/ophys_lims_api.py index e8a156a8e..e6683ced5 100644 --- a/allensdk/internal/api/ophys_lims_api.py +++ b/allensdk/internal/api/ophys_lims_api.py @@ -418,10 +418,20 @@ def get_raw_cell_specimen_table_dict(self): def get_cell_specimen_table(self): cell_specimen_table = pd.DataFrame.from_dict(self.get_raw_cell_specimen_table_dict()).set_index('cell_roi_id').sort_index() fov_width, fov_height = self.get_field_of_view_shape()['width'], self.get_field_of_view_shape()['height'] + + # Convert cropped ROI masks to uncropped versions image_mask_list = [] - for sub_mask in cell_specimen_table['image_mask'].values: - curr_roi = roi.create_roi_mask(fov_width, fov_height, [(fov_width - 1), 0, (fov_height - 1), 0], roi_mask=np.array(sub_mask, dtype=np.bool)) + for cell_roi_id, table_row in cell_specimen_table.iterrows(): + # Deserialize roi data into AllenSDK RoiMask object + curr_roi = roi.RoiMask(image_w=fov_width, image_h=fov_height, + label=None, mask_group=-1) + curr_roi.x = table_row['x'] + curr_roi.y = table_row['y'] + curr_roi.width = table_row['width'] + curr_roi.height = table_row['height'] + curr_roi.mask = np.array(table_row['image_mask']) image_mask_list.append(curr_roi.get_mask_plane().astype(np.bool)) + cell_specimen_table['image_mask'] = image_mask_list cell_specimen_table = cell_specimen_table[sorted(cell_specimen_table.columns)] diff --git a/allensdk/test/brain_observatory/behavior/conftest.py b/allensdk/test/brain_observatory/behavior/conftest.py index eafb4646f..d91e79f75 100644 --- a/allensdk/test/brain_observatory/behavior/conftest.py +++ b/allensdk/test/brain_observatory/behavior/conftest.py @@ -117,7 +117,7 @@ def stimulus_presentations_behavior(stimulus_templates, stimulus_presentations): @pytest.fixture def metadata(): - + """Fixture that passes all possible behavior ophys session metadata""" return {"ophys_experiment_id": 1234, "experiment_container_id": 5678, "ophys_frame_rate": 31.0, @@ -134,8 +134,31 @@ def metadata(): "emission_lambda": 1.0, "excitation_lambda": 1.0, "indicator": 'HW', - "field_of_view_width": 2, - "field_of_view_height": 2, + "field_of_view_width": 4, + "field_of_view_height": 4, + "rig_name": 'my_device', + "sex": 'M', + "age": 'P139', + } + + +@pytest.fixture +def partial_metadata(): + """Fixture that passes only metadata that will be saved in + custom pyNWB extension fields""" + return {"ophys_experiment_id": 1234, + "experiment_container_id": 5678, + "stimulus_frame_rate": 60.0, + "imaging_depth": 375, + "session_type": 'Unknown', + "experiment_datetime": pytz.utc.localize(datetime.datetime.now()), + "reporter_line": ["Ai93(TITL-GCaMP6f)"], + "driver_line": ["Camk2a-tTA", "Slc17a7-IRES2-Cre"], + "LabTracks_ID": 416369, + "full_genotype": "Slc17a7-IRES2-Cre/wt;Camk2a-tTA/wt;Ai93(TITL-GCaMP6f)/wt", + "behavior_session_uuid": uuid.uuid4(), + "field_of_view_width": 4, + "field_of_view_height": 4, "rig_name": 'my_device', "sex": 'M', "age": 'P139', @@ -160,20 +183,25 @@ def task_parameters(): @pytest.fixture def cell_specimen_table(): + return pd.DataFrame({'cell_roi_id': [123, 321], - 'x': [1, 1], - 'y': [1, 1], - 'width': [1, 1], - 'height': [1, 1], - 'valid_roi':[True, False], - 'max_correction_up':[1., 1.], - 'max_correction_down':[1., 1.], - 'max_correction_left':[1., 1.], - 'max_correction_right':[1., 1.], - 'mask_image_plane':[1, 1], - 'ophys_cell_segmentation_run_id':[1, 1], - 'image_mask': [np.array([[True, True], [False, False]]), np.array([[True, True], [False, False]])]}, - index=pd.Index([None, None], dtype=int, name='cell_specimen_id')) + 'x': [0, 2], + 'y': [0, 2], + 'width': [2, 2], + 'height': [2, 2], + 'valid_roi': [True, False], + 'max_correction_up': [1., 1.], + 'max_correction_down': [1., 1.], + 'max_correction_left': [1., 1.], + 'max_correction_right': [1., 1.], + 'mask_image_plane': [1, 1], + 'ophys_cell_segmentation_run_id': [1, 1], + 'image_mask': [np.array([[True, True], + [True, False]]), + np.array([[True, True], + [False, True]])]}, + index=pd.Index([None, None], dtype=int, + name='cell_specimen_id')) @pytest.fixture diff --git a/allensdk/test/brain_observatory/behavior/test_metadata_processing.py b/allensdk/test/brain_observatory/behavior/test_metadata_processing.py index 96e01384a..dee5eda3c 100644 --- a/allensdk/test/brain_observatory/behavior/test_metadata_processing.py +++ b/allensdk/test/brain_observatory/behavior/test_metadata_processing.py @@ -1,7 +1,9 @@ +import pytest import numpy as np from allensdk.brain_observatory.behavior.metadata_processing import ( - get_task_parameters) + OPHYS_1_3_DESCRIPTION, OPHYS_2_DESCRIPTION, OPHYS_4_6_DESCRIPTION, + OPHYS_5_DESCRIPTION, get_task_parameters, get_expt_description) def test_get_task_parameters(): @@ -54,3 +56,28 @@ def test_get_task_parameters(): except (TypeError, ValueError): assert expected[k] == v assert list(actual.keys()) == list(expected.keys()) + + +@pytest.mark.parametrize("session_type, expected_description", [ + ("OPHYS_1_images_A", OPHYS_1_3_DESCRIPTION), + ("OPHYS_2_images_B", OPHYS_2_DESCRIPTION), + ("OPHYS_3_images_C", OPHYS_1_3_DESCRIPTION), + ("OPHYS_4_images_D", OPHYS_4_6_DESCRIPTION), + ("OPHYS_5_images_E", OPHYS_5_DESCRIPTION), + ("OPHYS_6_images_F", OPHYS_4_6_DESCRIPTION) +]) +def test_get_expt_description_with_valid_session_type(session_type, + expected_description): + obt = get_expt_description(session_type) + assert obt == expected_description + + +@pytest.mark.parametrize("session_type", [ + ("bogus_session_type"), + ("stuff"), + ("OPHYS_7") +]) +def test_get_expt_description_raises_with_invalid_session_type(session_type): + error_msg_match_phrase = r"Encountered an unknown session type*" + with pytest.raises(RuntimeError, match=error_msg_match_phrase): + _ = get_expt_description(session_type) diff --git a/allensdk/test/brain_observatory/behavior/test_write_nwb_behavior_ophys.py b/allensdk/test/brain_observatory/behavior/test_write_nwb_behavior_ophys.py index e6b8da563..5337c8050 100644 --- a/allensdk/test/brain_observatory/behavior/test_write_nwb_behavior_ophys.py +++ b/allensdk/test/brain_observatory/behavior/test_write_nwb_behavior_ophys.py @@ -1,11 +1,16 @@ -import pytest -import pandas as pd -import numpy as np import math +import warnings + +import numpy as np +import pandas as pd +import pynwb +import pytest import allensdk.brain_observatory.nwb as nwb -from allensdk.brain_observatory.behavior.behavior_ophys_api.behavior_ophys_nwb_api import BehaviorOphysNwbApi -from allensdk.brain_observatory.behavior.schemas import OphysBehaviorMetadataSchema, BehaviorTaskParametersSchema +from allensdk.brain_observatory.behavior.behavior_ophys_api.behavior_ophys_nwb_api import \ + BehaviorOphysNwbApi +from allensdk.brain_observatory.behavior.schemas import ( + BehaviorTaskParametersSchema, OphysBehaviorMetadataSchema) @pytest.mark.parametrize('roundtrip', [True, False]) @@ -163,20 +168,38 @@ def test_segmentation_mask_image(nwbfile, roundtrip, roundtripper, segmentation_ assert image_api.deserialize(segmentation_mask_image) == image_api.deserialize(obt.get_segmentation_mask_image()) +@pytest.mark.parametrize('test_partial_metadata', [True, False]) @pytest.mark.parametrize('roundtrip', [True, False]) -def test_add_metadata(nwbfile, roundtrip, roundtripper, metadata): - - nwb.add_metadata(nwbfile, metadata) +def test_add_partial_metadata(test_partial_metadata, roundtrip, roundtripper, + cell_specimen_table, metadata, partial_metadata): + + meta = partial_metadata if test_partial_metadata else metadata + nwbfile = pynwb.NWBFile( + session_description='asession', + identifier='afile', + session_start_time=meta['experiment_datetime'] + ) + nwb.add_metadata(nwbfile, meta) + if not test_partial_metadata: + nwb.add_cell_specimen_table(nwbfile, cell_specimen_table, meta) if roundtrip: obt = roundtripper(nwbfile, BehaviorOphysNwbApi) else: obt = BehaviorOphysNwbApi.from_nwbfile(nwbfile) - metadata_obt = obt.get_metadata() + if not test_partial_metadata: + metadata_obt = obt.get_metadata() + else: + with warnings.catch_warnings(record=True) as record: + metadata_obt = obt.get_metadata() + exp_warn_msg = "Could not locate 'ophys' module in NWB" + print(record) + + assert record[0].message.args[0].startswith(exp_warn_msg) - assert len(metadata_obt) == len(metadata) - for key, val in metadata.items(): + assert len(metadata_obt) == len(meta) + for key, val in meta.items(): assert val == metadata_obt[key] @@ -208,7 +231,7 @@ def test_add_task_parameters(nwbfile, roundtrip, roundtripper, task_parameters): def test_get_cell_specimen_table(nwbfile, roundtrip, filter_invalid_rois, valid_roi_ids, roundtripper, cell_specimen_table, metadata, ophys_timestamps): nwb.add_metadata(nwbfile, metadata) - nwb.add_cell_specimen_table(nwbfile, cell_specimen_table) + nwb.add_cell_specimen_table(nwbfile, cell_specimen_table, metadata) if roundtrip: obt = roundtripper(nwbfile, BehaviorOphysNwbApi, filter_invalid_rois=filter_invalid_rois) @@ -226,7 +249,7 @@ def test_get_cell_specimen_table(nwbfile, roundtrip, filter_invalid_rois, valid_ def test_get_dff_traces(nwbfile, roundtrip, filter_invalid_rois, valid_roi_ids, roundtripper, dff_traces, cell_specimen_table, metadata, ophys_timestamps): nwb.add_metadata(nwbfile, metadata) - nwb.add_cell_specimen_table(nwbfile, cell_specimen_table) + nwb.add_cell_specimen_table(nwbfile, cell_specimen_table, metadata) nwb.add_dff_traces(nwbfile, dff_traces, ophys_timestamps) if roundtrip: @@ -245,7 +268,7 @@ def test_get_dff_traces(nwbfile, roundtrip, filter_invalid_rois, valid_roi_ids, def test_get_corrected_fluorescence_traces(nwbfile, roundtrip, filter_invalid_rois, valid_roi_ids, roundtripper, dff_traces, corrected_fluorescence_traces, cell_specimen_table, metadata, ophys_timestamps): nwb.add_metadata(nwbfile, metadata) - nwb.add_cell_specimen_table(nwbfile, cell_specimen_table) + nwb.add_cell_specimen_table(nwbfile, cell_specimen_table, metadata) nwb.add_dff_traces(nwbfile, dff_traces, ophys_timestamps) nwb.add_corrected_fluorescence_traces(nwbfile, corrected_fluorescence_traces) @@ -264,7 +287,7 @@ def test_get_corrected_fluorescence_traces(nwbfile, roundtrip, filter_invalid_ro def test_get_motion_correction(nwbfile, roundtrip, roundtripper, motion_correction, ophys_timestamps, metadata, cell_specimen_table, dff_traces): nwb.add_metadata(nwbfile, metadata) - nwb.add_cell_specimen_table(nwbfile, cell_specimen_table) + nwb.add_cell_specimen_table(nwbfile, cell_specimen_table, metadata) nwb.add_dff_traces(nwbfile, dff_traces, ophys_timestamps) nwb.add_motion_correction(nwbfile, motion_correction) diff --git a/requirements.txt b/requirements.txt index 3da662766..32ec2a495 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,7 +15,8 @@ scikit-image>=0.14.0,<0.17.0 scikit-build<1.0.0 statsmodels==0.9.0 simpleitk<2.0.0 -argschema==2.0.1 +argschema<2.0.0 +marshmallow==3.0.0rc6 glymur==0.8.19 xarray<0.16.0 pynwb>=1.3.2,<2.0.0