diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..31fe9fc --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,39 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: 'bug' +assignees: '' + +--- + +## Bug Report + +### Description + +A clear and concise description of what is the overall operation that is intended to be +performed that resulted in an error. + +### Reproducibility +Include: +- OS (WIN | MACOS | Linux) +- DataJoint Element Version +- MySQL Version +- MySQL Deployment Strategy (local-native | local-docker | remote) +- Minimum number of steps to reliably reproduce the issue +- Complete error stack as a result of evaluating the above steps + +### Expected Behavior +A clear and concise description of what you expected to happen. + +### Screenshots +If applicable, add screenshots to help explain your problem. + +### Additional Research and Context +Add any additional research or context that was conducted in creating this report. + +For example: +- Related GitHub issues and PR's either within this repository or in other relevant + repositories. +- Specific links to specific lines or a focus within source code. +- Relevant summary of Maintainers development meetings, milestones, projects, etc. diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..d31fbac --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,5 @@ +blank_issues_enabled: false +contact_links: + - name: DataJoint Contribution Guideline + url: https://docs.datajoint.org/python/community/02-Contribute.html + about: Please make sure to review the DataJoint Contribution Guidelines \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..1f2b784 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,57 @@ +--- +name: Feature request +about: Suggest an idea for a new feature +title: '' +labels: 'enhancement' +assignees: '' + +--- + +## Feature Request + +### Problem + +A clear and concise description how this idea has manifested and the context. Elaborate +on the need for this feature and/or what could be improved. Ex. I'm always frustrated +when [...] + +### Requirements + +A clear and concise description of the requirements to satisfy the new feature. Detail +what you expect from a successful implementation of the feature. Ex. When using this +feature, it should [...] + +### Justification + +Provide the key benefits in making this a supported feature. Ex. Adding support for this +feature would ensure [...] + +### Alternative Considerations + +Do you currently have a work-around for this? Provide any alternative solutions or +features you've considered. + +### Related Errors +Add any errors as a direct result of not exposing this feature. + +Please include steps to reproduce provided errors as follows: +- OS (WIN | MACOS | Linux) +- DataJoint Element Version +- MySQL Version +- MySQL Deployment Strategy (local-native | local-docker | remote) +- Minimum number of steps to reliably reproduce the issue +- Complete error stack as a result of evaluating the above steps + +### Screenshots +If applicable, add screenshots to help explain your feature. + +### Additional Research and Context +Add any additional research or context that was conducted in creating this feature request. + +For example: +- Related GitHub issues and PR's either within this repository or in other relevant + repositories. +- Specific links to specific lines or a focus within source code. +- Relevant summary of Maintainers development meetings, milestones, projects, etc. +- Any additional supplemental web references or links that would further justify this + feature request. diff --git a/CHANGELOG.md b/CHANGELOG.md index 1f205b6..9272141 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,13 +3,17 @@ Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) convention. ## [0.3.0] - 2022-10-7 + + Add - Function `prairieviewreader` to parse metadata from Bruker PrarieView acquisition system + ## 0.2.1 - 2022-07-13 + + Add - Adopt `black` formatting + Add - Code of Conduct ## 0.2.0 - 2022-07-06 + + First release of `element-interface`. + Bugfix - Fix for `tifffile` import. + Add - Function `run_caiman` to trigger CNMF algorithm. @@ -19,9 +23,11 @@ Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and + Update - Remove `extras_require` feature to allow this package to be published to PyPI. ## 0.1.0a1 - 2022-01-12 + + Change - Rename the package `element-data-loader` to `element-interface`. ## 0.1.0a0 - 2021-06-21 + + Add - Readers for: `ScanImage`, `Suite2p`, `CaImAn`. [0.3.0]: https://github.com/datajoint/element-interface/releases/tag/0.3.0 diff --git a/README.md b/README.md index f267a19..bf9f21c 100644 --- a/README.md +++ b/README.md @@ -60,27 +60,24 @@ repositories for example usage of `element-interface`. loaded_dataset = suite2p_loader.Suite2p(output_dir) ``` - + Suite2p wrapper functions for triggering analysis - Each step of Suite2p (registration, segmentation and deconvolution) - can be run independently for single plane tiff files. The functions in this - package will facilitate this process. Requirements include the [ops dictionary]( - https://suite2p.readthedocs.io/en/latest/settings.html) and db dictionary. - These wrapper functions were developed primarily because `run_s2p` cannot - individually run deconvolution using the `spikedetect` flag - ([Suite2p Issue #718](https://github.com/MouseLand/suite2p/issues/718)). + + Functions to independently run Suite2p's motion correction, segmentation, and deconvolution steps. These functions currently work for single plane tiff files. If running all Suite2p pre-processing steps concurrently, these functions are not required and one can run `suite2p.run_s2p()`. - ```python - from element_data_loader.suite2p_trigger import motion_correction_suite2p, - segmentation_suite2p, deconvolution_suite2p - from suite2p import default_ops + + These wrapper functions were developed primarily because `run_s2p` cannot individually run deconvolution using the `spikedetect` flag ([Suite2p Issue #718](https://github.com/MouseLand/suite2p/issues/718)). - ops = dict(default_ops(), nonrigid=False, two_step_registration=False) - ``` - Details of db dictionary can be found [here](https://github.com/MouseLand/suite2p/blob/4b6c3a95b53e5581dbab1feb26d67878db866068/jupyter/run_pipeline_tiffs_or_batch.ipynb) + + Requirements + + [ops dictionary](https://suite2p.readthedocs.io/en/latest/settings.html) + + [db dictionary](https://github.com/MouseLand/suite2p/blob/4b6c3a95b53e5581dbab1feb26d67878db866068/jupyter/run_pipeline_tiffs_or_batch.ipynb) + + + Note that the ops dictionary returned from the `motion_correction_suite2p` and `segmentation_suite2p` functions is only a subset of the keys generated with the `suite2p.default_ops()` function. ```python + import element_interface + import suite2p + + ops = dict(suite2p.default_ops(), nonrigid=False, two_step_registration=False) + db = { 'h5py': [], # single h5 file path 'h5py_key': 'data', @@ -91,13 +88,13 @@ repositories for example usage of `element-interface`. } ops.update(do_registration=1, roidetect=False, spikedetect=False) - registration_ops = motion_registration_suite2p(ops, db) + motion_correction_ops = element_interface.suite2p_trigger.motion_correction_suite2p(ops, db) - registration_ops.update(do_registration=0, roidetect=True, spikedetect=False) - segmentation_ops = segmentation_suite2p(registration_ops, db) + motion_correction_ops.update(do_registration=0, roidetect=True, spikedetect=False) + segmentation_ops = element_interface.suite2p_trigger.segmentation_suite2p(motion_correction_ops, db) segmentation_ops.update(do_registration=0, roidetect=False, spikedetect=True) - spikes = deconvolution_suite2p(segmentation_ops, db) + spikes = element_interface.suite2p_trigger.deconvolution_suite2p(segmentation_ops, db) ``` diff --git a/element_interface/suite2p_trigger.py b/element_interface/suite2p_trigger.py index 555ec00..aae4f5e 100644 --- a/element_interface/suite2p_trigger.py +++ b/element_interface/suite2p_trigger.py @@ -1,37 +1,29 @@ -from suite2p import run_s2p -from suite2p.extraction import dcnv +import suite2p import numpy as np import os import warnings def motion_correction_suite2p(ops, db): - """Performs registration using Suite2p package - + """Performs motion correction (i.e. registration) using the Suite2p package. Args: - ops (dict): ops dictionary can be obtained by using default_ops() + ops (dict): ops dictionary can be obtained by using `suite2p.default_ops()` function. It contains all options and default values used to perform preprocessing. ops['do_registration'] should be set to 1. db (dict): dictionary that includes paths pointing towards the input data, and path to store outputs - Returns: - registration_ops (dict): Returns a dictionary that includes x and y shifts. - This dictionary only consists of a subset of the - output so that we only use the required values - in the segmentation step. - data.bin: Creates and saves a binary file on your local path. If - delete_bin is set to True (default: False), the binary file is - deleted after processing. - ops.npy: Creates and saves the options dictionary in a numpy array file - in the specified path. The ops file gets updated during the - cell detection and the deconvolution steps. + motion_correction_ops (dict): Returns a dictionary that includes x and y shifts. + A subset of the ops dictionary returned from `suite2p.run_s2p()` that is required for the segmentation step. + data.bin: Binary file of the data. If delete_bin is set to True (default: False), the binary file is deleted after processing. + ops.npy: Options dictionary. This file gets updated during the + segmentation and deconvolution steps. """ if (not ops["do_registration"]) or ops["roidetect"] or ops["spikedetect"]: warnings.warn( - "You are running motion correction with Suite2p." + "Running motion correction with Suite2p." "Requirements include do_registration=1," "roidetect=False, spikedetect=False. The ops" "dictionary has differing values. The flags will" @@ -42,9 +34,9 @@ def motion_correction_suite2p(ops, db): if ops["nonrigid"]: - print("------------RUNNING NON-RIGID REGISTRATION------------") + print("------------Running non-rigid motion correction------------") - registration_ops = run_s2p(ops, db) + motion_correction_ops = suite2p.run_s2p(ops, db) subset_keys = [ "xoff", "yoff", @@ -65,9 +57,9 @@ def motion_correction_suite2p(ops, db): else: - print("------------RUNNING RIGID REGISTRATION------------") + print("------------Running rigid motion correction------------") - registration_ops = run_s2p(ops, db) + motion_correction_ops = suite2p.run_s2p(ops, db) subset_keys = [ "xoff", "yoff", @@ -78,59 +70,49 @@ def motion_correction_suite2p(ops, db): "delete_bin", ] - registration_ops = {key: registration_ops[key] for key in subset_keys} + motion_correction_ops = {key: motion_correction_ops[key] for key in subset_keys} - return registration_ops + return motion_correction_ops -def segmentation_suite2p(registration_ops, db): - """Performs cell detection using Suite2p package, - Make sure the 'roidetect' key is set to True to perform cell/ roi detection - +def segmentation_suite2p(motion_correction_ops, db): + """Performs cell segmentation (i.e. roi detection) using Suite2p package. Args: - registration_ops (dict): reg_ops dictionary can be obtained from - registration step (CaImAn or Suite2p). - Must contain x and y shifts along with - 'do_registration'=0, - 'two_step_registration'=False, - 'roidetect'=True and 'spikedetect'= False + motion_correction_ops (dict): options dictionary. Requirements: + - x and y shifts + - do_registration=0 + - two_step_registration=False + - roidetect=True + - spikedetect=False db (dict): dictionary that includes paths pointing towards the input data, and path to store outputs - Returns: - segmentation_ops (dict): The ops dictionary is updated with additional - keys that are added. This dictionary only - consists of a subset of the - output so that we only use the required values - in the deconvolution step. - data.bin: Creates and saves a binary file on your local path, if the one - created during registration is deleted. If delete_bin is set - to True, the binary file is deleted after processing. - ops.npy: Updates the options dictionary in a file created during the - registration step and saves as a numpy array file on the - specified path. - F.npy: Stores an array of fluorescence traces - Fneu.npy: Returns an array of neuropil fluorescence traces - iscell.npy: Specifies whether an ROI is a cell - stat.npy: Returns a list of statistics computed for each cell - spks.npy: Returns an empty file, it is updated during the deconvolution - step with deconvolved traces + segmentation_ops (dict): A subset of the ops dictionary returned from `suite2p.run_s2p()` that is required for the deconvolution step. + data.bin: Binary file if the one created during motion correction is deleted. If delete_bin=True, the binary file is deleted after processing. + ops.npy: Updated ops dictionary created by suite2p.run_s2p() + F.npy: Array of fluorescence traces + Fneu.npy: Array of neuropil fluorescence traces + iscell.npy: Specifies whether a region of interest is a cell and the probability + stat.npy: List of statistics computed for each cell + spks.npy: Empty file. This file is updated with deconvolved traces during the deconvolution step. """ if ( - registration_ops["do_registration"] - or not (registration_ops["roidetect"]) - or registration_ops["spikedetect"] + motion_correction_ops["do_registration"] + or not motion_correction_ops["roidetect"] + or motion_correction_ops["spikedetect"] ): warnings.warn( - "You are running segmentation with Suite2p. Requirements" + "Running segmentation with Suite2p. Requirements" "include do_registration=0, roidetect=True," "spikedetect=False. The ops dictionary has differing" "values. The flags will be set to the required values." ) - registration_ops.update(do_registration=0, roidetect=True, spikedetect=False) + motion_correction_ops.update( + do_registration=0, roidetect=True, spikedetect=False + ) - segmentation_ops = run_s2p(registration_ops, db) + segmentation_ops = suite2p.run_s2p(motion_correction_ops, db) subset_keys = [ "baseline", "win_baseline", @@ -152,23 +134,23 @@ def segmentation_suite2p(registration_ops, db): def deconvolution_suite2p(segmentation_ops, db): - """Performs cell detection using Suite2p package, - Make sure the 'roidetect' key is set to True to perform cell/ roi detection. + """Performs deconvolution using the Suite2p package for single plane tiff files. The code to run deconvolution separately can be found here . - Args: - segmentation_ops (dict): roi_ops dictionary can be obtained from registration - step (CaImAn or Suite2p). Must contain baseline, - win_baseline (window in seconds for max filter), - sig_baseline (width of Gaussian filter in seconds), - fs (sampling rate per plane), - prctile_baseline (percentile of trace to use as baseline - is using constant_prctile for baseline) - along with 'do_registration'=0, - 'two_step_registration'=False,'roidetect'=False and - 'spikedetect'= True - + segmentation_ops (dict): options dictionary. Requirements: + - baseline - setting that describes how to compute the baseline of each trace + - win_baseline - window for max filter in seconds + - sig_baseline - width of Gaussian filter in seconds + - fs - sampling rate per plane + - prctile_baseline - percentile of trace to use as baseline if using `constant_prctile` for baseline + - batch_size - number of frames processed per batch + - tau - timescale of the sensor, used for the deconvolution kernel + - neucoeff - neuropil coefficient for all regions of interest + - do_registration=0 + - two_step_registration=False + - roidetect=False + - spikedetect=True Returns: spks.npy: Updates the file with an array of deconvolved traces """ @@ -178,10 +160,10 @@ def deconvolution_suite2p(segmentation_ops, db): or (not segmentation_ops["spikedetect"]) ): warnings.warn( - "You are running deconvolution using Suite2p module." + "Running deconvolution with Suite2p." "Requirements include do_registration=0, roidetect=False," "spikedetect=True. The ops dictionary has differing values," - "the flags will be se set to the required values." + "the flags will be set to the required values." ) segmentation_ops.update(do_registration=0, roidetect=False, spikedetect=True) @@ -191,7 +173,7 @@ def deconvolution_suite2p(segmentation_ops, db): ) Fc = F - segmentation_ops["neucoeff"] * Fneu - Fc = dcnv.preprocess( + Fc = suite2p.extraction.dcnv.preprocess( F=Fc, baseline=segmentation_ops["baseline"], win_baseline=segmentation_ops["win_baseline"], @@ -200,7 +182,7 @@ def deconvolution_suite2p(segmentation_ops, db): prctile_baseline=segmentation_ops["prctile_baseline"], ) - spikes = dcnv.oasis( + spikes = suite2p.extraction.dcnv.oasis( F=Fc, batch_size=segmentation_ops["batch_size"], tau=segmentation_ops["tau"], diff --git a/element_interface/version.py b/element_interface/version.py index 5ca6608..0697f8a 100644 --- a/element_interface/version.py +++ b/element_interface/version.py @@ -1,2 +1,3 @@ """Package metadata""" + __version__ = "0.3.0" diff --git a/requirements.txt b/requirements.txt index 819f637..bc4c980 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ numpy==1.21.0 +dandi \ No newline at end of file