From c74f89b2ed048c18a02f339ae7dd5496027e6803 Mon Sep 17 00:00:00 2001 From: kushalbakshi Date: Tue, 11 Jul 2023 15:07:17 -0500 Subject: [PATCH 1/5] Update pathlib error in PrairieViewReader --- element_interface/prairieviewreader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/element_interface/prairieviewreader.py b/element_interface/prairieviewreader.py index 31e0170..146a2d3 100644 --- a/element_interface/prairieviewreader.py +++ b/element_interface/prairieviewreader.py @@ -38,7 +38,7 @@ def get_pv_metadata(pvtiffile: str) -> dict: break else: raise FileNotFoundError( - f"No PrarieView metadata XML file found at {pvtiffile.parent}" + f"No PrarieView metadata XML file found at {pathlib.Path(pvtiffile).parent}" ) bidirectional_scan = False # Does not support bidirectional From 593279940590e3f7f9888df928bb41c371bbba84 Mon Sep 17 00:00:00 2001 From: kushalbakshi Date: Wed, 26 Jul 2023 10:33:39 -0500 Subject: [PATCH 2/5] Update internal variable names in prairieviewreader + CHANGELOG + version --- CHANGELOG.md | 6 ++ element_interface/prairieviewreader.py | 120 +++++++++++++------------ element_interface/version.py | 2 +- 3 files changed, 72 insertions(+), 56 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 41d68ed..1d52786 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,11 @@ Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) convention. +## [0.5.5] - 2023-07-26 + ++ Update - `prairieviewreader.py` -> `prairie_view_loader.py` ++ Update - Internal variable names within `prairie_view_loader.py` + ## [0.5.4] - 2023-05-14 + Fix - DANDI URL for uploads where `staging=False`. @@ -70,6 +75,7 @@ Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and + Add - Readers for: `ScanImage`, `Suite2p`, `CaImAn`. +[0.5.5]: https://github.com/datajoint/element-interface/releases/tag/0.5.5 [0.5.4]: https://github.com/datajoint/element-interface/releases/tag/0.5.4 [0.5.3]: https://github.com/datajoint/element-interface/releases/tag/0.5.3 [0.5.2]: https://github.com/datajoint/element-interface/releases/tag/0.5.2 diff --git a/element_interface/prairieviewreader.py b/element_interface/prairieviewreader.py index 146a2d3..7fc082e 100644 --- a/element_interface/prairieviewreader.py +++ b/element_interface/prairieviewreader.py @@ -5,23 +5,23 @@ import numpy as np -def get_pv_metadata(pvtiffile: str) -> dict: - """Extract metadata for scans generated by PrairieView acquisition software. +def get_pv_metadata(ome_tif_filepath: str) -> dict: + """Extract metadata for scans generated by Prairie View acquisition software. - The PrairieView software generates one .ome.tif imaging file per frame acquired. The - metadata for all frames is contained one .xml file. This function locates the .xml - file and generates a dictionary necessary to populate the DataJoint ScanInfo and - Field tables. PrairieView works with resonance scanners with a single field. - PrairieView does not support bidirectional x and y scanning. ROI information is not - contained in the .xml file. All images generated using PrairieView have square - dimensions(e.g. 512x512). + The Prairie View software generates one `.ome.tif` imaging file per frame + acquired. The metadata for all frames is contained in one .xml file. This + function locates the .xml file and generates a dictionary necessary to + populate the DataJoint `ScanInfo` and `Field` tables. Prairie View works + with resonance scanners with a single field. Prairie View does not support + bidirectional x and y scanning. ROI information is not contained in the + `.xml` file. All images generated using PrairieView have square dimensions(e.g. 512x512). Args: - pvtiffile: An absolute path to the .ome.tif image file. + ome_tif_filepath: An absolute path to the .ome.tif image file. Raises: FileNotFoundError: No .xml file containing information about the acquired scan - was found at path in parent directory at `pvtiffile`. + was found at path in parent directory at `ome_tif_filepath`. Returns: metainfo: A dict mapping keys to corresponding metadata values fetched from the @@ -29,83 +29,86 @@ def get_pv_metadata(pvtiffile: str) -> dict: """ # May return multiple xml files. Only need one that contains scan metadata. - xml_files = pathlib.Path(pvtiffile).parent.glob("*.xml") + xml_files_list = pathlib.Path(ome_tif_filepath).parent.glob("*.xml") - for xml_file in xml_files: - tree = ET.parse(xml_file) - root = tree.getroot() - if root.find(".//Sequence"): + for file in xml_files_list: + xml_tree = ET.parse(file) + xml_file = xml_tree.getroot() + if xml_file.find(".//Sequence"): break else: raise FileNotFoundError( - f"No PrarieView metadata XML file found at {pathlib.Path(pvtiffile).parent}" + f"No PrarieView metadata .xml file found at {pathlib.Path(ome_tif_filepath).parent}" ) bidirectional_scan = False # Does not support bidirectional roi = 0 n_fields = 1 # Always contains 1 field - record_start_time = root.find(".//Sequence/[@cycle='1']").attrib.get("time") + recording_start_time = xml_file.find(".//Sequence/[@cycle='1']").attrib.get("time") # Get all channels and find unique values channel_list = [ int(channel.attrib.get("channel")) - for channel in root.iterfind(".//Sequence/Frame/File/[@channel]") + for channel in xml_file.iterfind(".//Sequence/Frame/File/[@channel]") ] n_channels = len(set(channel_list)) - n_frames = len(root.findall(".//Sequence/Frame")) + n_frames = len(xml_file.findall(".//Sequence/Frame")) framerate = 1 / float( - root.findall('.//PVStateValue/[@key="framePeriod"]')[0].attrib.get("value") + xml_file.findall('.//PVStateValue/[@key="framePeriod"]')[0].attrib.get("value") ) # rate = 1/framePeriod usec_per_line = ( float( - root.findall(".//PVStateValue/[@key='scanLinePeriod']")[0].attrib.get( + xml_file.findall(".//PVStateValue/[@key='scanLinePeriod']")[0].attrib.get( "value" ) ) * 1e6 ) # Convert from seconds to microseconds - scan_datetime = datetime.strptime(root.attrib.get("date"), "%m/%d/%Y %I:%M:%S %p") + scan_datetime = datetime.strptime( + xml_file.attrib.get("date"), "%m/%d/%Y %I:%M:%S %p" + ) - total_duration = float( - root.findall(".//Sequence/Frame")[-1].attrib.get("relativeTime") + total_scan_duration = float( + xml_file.findall(".//Sequence/Frame")[-1].attrib.get("relativeTime") ) - px_height = int( - root.findall(".//PVStateValue/[@key='pixelsPerLine']")[0].attrib.get("value") + pixel_height = int( + xml_file.findall(".//PVStateValue/[@key='pixelsPerLine']")[0].attrib.get( + "value" + ) ) # All PrairieView-acquired images have square dimensions (512 x 512; 1024 x 1024) - px_width = px_height + pixel_width = pixel_height um_per_pixel = float( - root.find( + xml_file.find( ".//PVStateValue/[@key='micronsPerPixel']/IndexedValue/[@index='XAxis']" ).attrib.get("value") ) - um_height = um_width = float(px_height) * um_per_pixel + um_height = um_width = float(pixel_height) * um_per_pixel # x and y coordinate values for the center of the field x_field = float( - root.find( + xml_file.find( ".//PVStateValue/[@key='currentScanCenter']/IndexedValue/[@index='XAxis']" ).attrib.get("value") ) y_field = float( - root.find( + xml_file.find( ".//PVStateValue/[@key='currentScanCenter']/IndexedValue/[@index='YAxis']" ).attrib.get("value") ) if ( - root.find( + xml_file.find( ".//Sequence/[@cycle='1']/Frame/PVStateShard/PVStateValue/[@key='positionCurrent']/SubindexedValues/[@index='ZAxis']" ) is None ): - z_fields = np.float64( - root.find( + xml_file.find( ".//PVStateValue/[@key='positionCurrent']/SubindexedValues/[@index='ZAxis']/SubindexedValue" ).attrib.get("value") ) @@ -114,46 +117,53 @@ def get_pv_metadata(pvtiffile: str) -> dict: bidirection_z = False else: + bidirection_z = ( + xml_file.find(".//Sequence").attrib.get("bidirectionalZ") == "True" + ) - bidirection_z = root.find(".//Sequence").attrib.get("bidirectionalZ") == "True" - - # One "Frame" per depth. Gets number of frames in first sequence + # One "Frame" per depth in the .xml file. Gets number of frames in first sequence planes = [ int(plane.attrib.get("index")) - for plane in root.findall(".//Sequence/[@cycle='1']/Frame") + for plane in xml_file.findall(".//Sequence/[@cycle='1']/Frame") ] n_depths = len(set(planes)) - z_controllers = root.findall( + z_controllers = xml_file.findall( ".//Sequence/[@cycle='1']/Frame/[@index='1']/PVStateShard/PVStateValue/[@key='positionCurrent']/SubindexedValues/[@index='ZAxis']/SubindexedValue" ) - if len(z_controllers) > 1: + # If more than one Z-axis controllers are found, the code block below + # checks which controller is changing z_field depth. Only 1 controller + # must change depths. + if len(z_controllers) > 1: z_repeats = [] - for controller in root.findall( - ".//Sequence/[@cycle='1']/Frame/[@index='1']/PVStateShard/PVStateValue/[@key='positionCurrent']/SubindexedValues/[@index='ZAxis']/"): + for controller in xml_file.findall( + ".//Sequence/[@cycle='1']/Frame/[@index='1']/PVStateShard/PVStateValue/[@key='positionCurrent']/SubindexedValues/[@index='ZAxis']/" + ): z_repeats.append( [ float(z.attrib.get("value")) - for z in root.findall( + for z in xml_file.findall( ".//Sequence/[@cycle='1']/Frame/PVStateShard/PVStateValue/[@key='positionCurrent']/SubindexedValues/[@index='ZAxis']/SubindexedValue/[@subindex='{0}']".format( controller.attrib.get("subindex") ) ) ] ) - - - controller_assert = [not all(z == z_controller[0] for z in z_controller) for z_controller in z_repeats] - - assert sum(controller_assert)==1, "Multiple controllers changing z depth is not supported" + controller_assert = [ + not all(z == z_controller[0] for z in z_controller) + for z_controller in z_repeats + ] + assert ( + sum(controller_assert) == 1 + ), "Multiple controllers changing z depth is not supported" z_fields = z_repeats[controller_assert.index(True)] - + else: z_fields = [ z.attrib.get("value") - for z in root.findall( + for z in xml_file.findall( ".//Sequence/[@cycle='1']/Frame/PVStateShard/PVStateValue/[@key='positionCurrent']/SubindexedValues/[@index='ZAxis']/SubindexedValue/[@subindex='0']" ) ] @@ -176,15 +186,15 @@ def get_pv_metadata(pvtiffile: str) -> dict: bidirectional_z=bidirection_z, scan_datetime=scan_datetime, usecs_per_line=usec_per_line, - scan_duration=total_duration, - height_in_pixels=px_height, - width_in_pixels=px_width, + scan_duration=total_scan_duration, + height_in_pixels=pixel_height, + width_in_pixels=pixel_width, height_in_um=um_height, width_in_um=um_width, fieldX=x_field, fieldY=y_field, fieldZ=z_fields, - recording_time=record_start_time, + recording_time=recording_start_time, ) return metainfo diff --git a/element_interface/version.py b/element_interface/version.py index edb1b12..a5ad586 100644 --- a/element_interface/version.py +++ b/element_interface/version.py @@ -1,3 +1,3 @@ """Package metadata""" -__version__ = "0.5.4" +__version__ = "0.5.5" From d66a0ca70f1cd1ece8f211767d9db0f8f804356f Mon Sep 17 00:00:00 2001 From: Kushal Bakshi <52367253+kushalbakshi@users.noreply.github.com> Date: Wed, 26 Jul 2023 12:39:15 -0500 Subject: [PATCH 3/5] Apply suggestions from code review Co-authored-by: Kabilar Gunalan --- CHANGELOG.md | 4 ++-- element_interface/prairieviewreader.py | 8 ++++---- element_interface/version.py | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ca3b874..d670a3d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) convention. -## [0.5.5] - 2023-07-26 +## [0.6.0] - 2023-07-26 + Update - `prairieviewreader.py` -> `prairie_view_loader.py` + Update - Internal variable names within `prairie_view_loader.py` @@ -76,7 +76,7 @@ Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and + Add - Readers for: `ScanImage`, `Suite2p`, `CaImAn`. -[0.5.5]: https://github.com/datajoint/element-interface/releases/tag/0.5.5 +[0.6.0]: https://github.com/datajoint/element-interface/releases/tag/0.6.0 [0.5.4]: https://github.com/datajoint/element-interface/releases/tag/0.5.4 [0.5.3]: https://github.com/datajoint/element-interface/releases/tag/0.5.3 [0.5.2]: https://github.com/datajoint/element-interface/releases/tag/0.5.2 diff --git a/element_interface/prairieviewreader.py b/element_interface/prairieviewreader.py index 7fc082e..841e87a 100644 --- a/element_interface/prairieviewreader.py +++ b/element_interface/prairieviewreader.py @@ -5,7 +5,7 @@ import numpy as np -def get_pv_metadata(ome_tif_filepath: str) -> dict: +def get_prairieview_metadata(ome_tif_filepath: str) -> dict: """Extract metadata for scans generated by Prairie View acquisition software. The Prairie View software generates one `.ome.tif` imaging file per frame @@ -14,7 +14,7 @@ def get_pv_metadata(ome_tif_filepath: str) -> dict: populate the DataJoint `ScanInfo` and `Field` tables. Prairie View works with resonance scanners with a single field. Prairie View does not support bidirectional x and y scanning. ROI information is not contained in the - `.xml` file. All images generated using PrairieView have square dimensions(e.g. 512x512). + `.xml` file. All images generated using Prairie View have square dimensions(e.g. 512x512). Args: ome_tif_filepath: An absolute path to the .ome.tif image file. @@ -132,8 +132,8 @@ def get_pv_metadata(ome_tif_filepath: str) -> dict: ".//Sequence/[@cycle='1']/Frame/[@index='1']/PVStateShard/PVStateValue/[@key='positionCurrent']/SubindexedValues/[@index='ZAxis']/SubindexedValue" ) - # If more than one Z-axis controllers are found, the code block below - # checks which controller is changing z_field depth. Only 1 controller + # If more than one Z-axis controllers are found, + # check which controller is changing z_field depth. Only 1 controller # must change depths. if len(z_controllers) > 1: z_repeats = [] diff --git a/element_interface/version.py b/element_interface/version.py index a5ad586..6d90ece 100644 --- a/element_interface/version.py +++ b/element_interface/version.py @@ -1,3 +1,3 @@ """Package metadata""" -__version__ = "0.5.5" +__version__ = "0.6.0" From da4c1c73b59c7bd31804d85d4499394fc55eecb2 Mon Sep 17 00:00:00 2001 From: kushalbakshi Date: Wed, 26 Jul 2023 12:42:21 -0500 Subject: [PATCH 4/5] Rename `prairieviewreader.py` -> `prairie_view_loader.py` --- .../{prairieviewreader.py => prairie_view_loader.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename element_interface/{prairieviewreader.py => prairie_view_loader.py} (100%) diff --git a/element_interface/prairieviewreader.py b/element_interface/prairie_view_loader.py similarity index 100% rename from element_interface/prairieviewreader.py rename to element_interface/prairie_view_loader.py From b34a517eeed7df83043d677cd3fdd41660538136 Mon Sep 17 00:00:00 2001 From: Kushal Bakshi <52367253+kushalbakshi@users.noreply.github.com> Date: Thu, 27 Jul 2023 08:11:25 -0500 Subject: [PATCH 5/5] Update CHANGELOG.md Co-authored-by: Kabilar Gunalan --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d670a3d..7293361 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and ## [0.6.0] - 2023-07-26 + Update - `prairieviewreader.py` -> `prairie_view_loader.py` ++ Update - `get_pv_metadata()` -> `get_prairieview_metadata()` + Update - Internal variable names within `prairie_view_loader.py` ## [0.5.4] - 2023-05-25