From 3e198ca9994f03e238c42bced47febb341925214 Mon Sep 17 00:00:00 2001 From: Luigi Petrucco Date: Mon, 1 Jun 2020 11:03:37 +0200 Subject: [PATCH] Centralising all atlas saving --- atlas_gen/atlas_scripts/example_atlas.py | 5 -- atlas_gen/metadata.py | 47 ----------- atlas_gen/metadata_utils.py | 64 +++++++++++++-- atlas_gen/stacks.py | 2 +- atlas_gen/wrapup.py | 99 +++++++++++++----------- brainatlas_api/descriptors.py | 3 + 6 files changed, 114 insertions(+), 106 deletions(-) delete mode 100644 atlas_gen/metadata.py diff --git a/atlas_gen/atlas_scripts/example_atlas.py b/atlas_gen/atlas_scripts/example_atlas.py index 9769866..c4a6c76 100644 --- a/atlas_gen/atlas_scripts/example_atlas.py +++ b/atlas_gen/atlas_scripts/example_atlas.py @@ -7,7 +7,6 @@ import json import shutil -from atlas_gen.stacks import save_anatomy, save_annotation from atlas_gen.wrapup import wrapup_atlas_from_dir from brainatlas_api import descriptors @@ -46,10 +45,6 @@ template_volume, _ = spacecache.get_template_volume() print("Download completed...") -# Save tiff stacks: -save_anatomy(template_volume, uncompr_atlas_path) -save_annotation(annotated_volume, uncompr_atlas_path) - # Download structures tree and meshes: ###################################### oapi = OntologiesApi() # ontologies diff --git a/atlas_gen/metadata.py b/atlas_gen/metadata.py deleted file mode 100644 index 62df64d..0000000 --- a/atlas_gen/metadata.py +++ /dev/null @@ -1,47 +0,0 @@ -import requests -import re -from requests.exceptions import MissingSchema, InvalidURL, ConnectionError - - -def generate_metadata_dict( - name, citation, atlas_link, species, symmetric, resolution, version, shape -): - - # We ask for a rigid naming convention to be followed: - parsename = name.split("_") - assert len(parsename) >= 3 - assert re.match("[0-9]+um", parsename[-1]) - - # Control version formatting: - assert re.match("[0-9]+\\.[0-9]+", version) - - # We ask for DOI and correct link only if atlas is published: - if citation != "unpublished": - assert "doi" in citation - - # Test url: - try: - _ = requests.get(atlas_link) - except (MissingSchema, InvalidURL, ConnectionError): - raise InvalidURL( - "Ensure that the url is valid and formatted correctly!" - ) - - # Enforce correct format for symmetric, resolution and shape: - assert type(symmetric) == bool - assert len(resolution) == 3 - assert len(shape) == 3 - - resolution = tuple([float(v) for v in resolution]) - shape = tuple(int(v) for v in shape) - - return dict( - name=name, - citation=citation, - atlas_link=atlas_link, - species=species, - symmetric=symmetric, - resolution=resolution, - version=version, - shape=shape, - ) diff --git a/atlas_gen/metadata_utils.py b/atlas_gen/metadata_utils.py index 243408a..f0dd148 100644 --- a/atlas_gen/metadata_utils.py +++ b/atlas_gen/metadata_utils.py @@ -3,12 +3,60 @@ . structures.csv . README.txt """ +import re +import json from datetime import datetime +from brainatlas_api import descriptors + +import requests +from requests.exceptions import MissingSchema, InvalidURL, ConnectionError from brainatlas_api.structures.structure_tree import StructureTree -from brainatlas_api.atlas_gen.structure_json_to_csv import ( - convert_structure_json_to_csv, -) +from atlas_gen.structure_json_to_csv import convert_structure_json_to_csv + + +def generate_metadata_dict( + name, citation, atlas_link, species, symmetric, resolution, version, shape +): + + # We ask for a rigid naming convention to be followed: + parsename = name.split("_") + assert len(parsename) >= 3 + assert re.match("[0-9]+um", parsename[-1]) + + # Control version formatting: + assert re.match("[0-9]+\\.[0-9]+", version) + + # We ask for DOI and correct link only if atlas is published: + if citation != "unpublished": + assert "doi" in citation + + # Test url: + try: + _ = requests.get(atlas_link) + except (MissingSchema, InvalidURL, ConnectionError): + raise InvalidURL( + "Ensure that the url is valid and formatted correctly!" + ) + + # Enforce correct format for symmetric, resolution and shape: + assert type(symmetric) == bool + assert len(resolution) == 3 + assert len(shape) == 3 + + resolution = tuple([float(v) for v in resolution]) + shape = tuple(int(v) for v in shape) + + return dict( + name=name, + citation=citation, + atlas_link=atlas_link, + species=species, + symmetric=symmetric, + resolution=resolution, + version=version, + shape=shape, + ) def create_readme(uncompr_atlas_path, metadata_dict, structures): @@ -59,7 +107,7 @@ def create_structures_csv(uncompr_atlas_path, root): ) -def create_metadata_files(uncompr_atlas_path, metadata_dict, structures, root): +def create_metadata_files(dest_dir, metadata_dict, structures, root_id): """ Automatic creation of . structures.csv @@ -71,5 +119,9 @@ def create_metadata_files(uncompr_atlas_path, metadata_dict, structures, root): :param metadata_dict: dict with atlas metadata :param structures: list of dictionaries with structures hierarchical info """ - create_structures_csv(uncompr_atlas_path, root) - create_readme(uncompr_atlas_path, metadata_dict, structures) + # write metadata dict: + with open(dest_dir / descriptors.METADATA_FILENAME, "w") as f: + json.dump(metadata_dict, f) + + create_structures_csv(dest_dir, root_id) + create_readme(dest_dir, metadata_dict, structures) diff --git a/atlas_gen/stacks.py b/atlas_gen/stacks.py index b8ed9a4..4b2bd3f 100644 --- a/atlas_gen/stacks.py +++ b/atlas_gen/stacks.py @@ -13,7 +13,7 @@ def write_stack(stack, filename): tifffile.imsave(str(filename), stack) -def save_anatomy(stack, output_dir): +def save_reference(stack, output_dir): """ Parameters ---------- diff --git a/atlas_gen/wrapup.py b/atlas_gen/wrapup.py index 8c2c944..a627749 100644 --- a/atlas_gen/wrapup.py +++ b/atlas_gen/wrapup.py @@ -1,31 +1,47 @@ import json -from brainatlas_api.atlas_gen.metadata_utils import create_metadata_files -from brainatlas_api.utils import read_tiff, read_json -from .metadata import generate_metadata_dict -from .structures import check_struct_consistency +from atlas_gen.metadata_utils import ( + create_metadata_files, + generate_metadata_dict, +) +from atlas_gen.stacks import save_reference, save_annotation + +# from brainatlas_api.utils import read_tiff, read_json +from atlas_gen.structures import check_struct_consistency from brainatlas_api import descriptors import tarfile import shutil +# import bgspace as bgs -def wrapup_atlas_from_dir( - dir_path, +# This should be changed every time we make changes in the atlas +# structure: +ATLAS_VERSION = 0 + + +def wrapup_atlas_from_data( + atlas_name, + atlas_minor_version, citation, atlas_link, species, resolution, + orientation, + root_id, + reference_stack, + annotation_stack, + structures_list, + meshes_dict, + working_dir, + hemispheres_stack=None, cleanup_files=False, compress=True, - root=997, ): """ - Check compliance of a folder with atlas standards, write metadata, and if required compress and cleanup. - This function should be used to finalize all atlases as it runs the required - controls. + Finalise an atlas with truly consistent format from all the data. Parameters ---------- - dir_path : str or Path object + dest_dir : str or Path object directory with the atlases and regions description citation : str citation for the atlas, if unpublished specify "unpublished" @@ -43,36 +59,14 @@ def wrapup_atlas_from_dir( """ - # Check that all core files are contained: - for element in [ - descriptors.STRUCTURES_FILENAME, - descriptors.REFERENCE_FILENAME, - descriptors.ANNOTATION_FILENAME, - ]: - assert (dir_path / element).exists() - - # Get name and version from dir name - in this way multiple - # specifications are avoided: - parsename = dir_path.name.split("_") - - atlas_name = "_".join(parsename[:-1]) - version = parsename[-1][1:] # version: v0.0 format - - # Read stack shape: - ref_stack = read_tiff(dir_path / descriptors.REFERENCE_FILENAME) - shape = ref_stack.shape - - # If no hemisphere file is given, ensure the atlas is symmetric: - if not (dir_path / descriptors.HEMISPHERES_FILENAME).exists(): - # assert np.allclose(ref_stack[:, :, :shape[2] // 2], - # np.flip(ref_stack[:, :, -shape[2] // 2:], 2)) - symmetric = True - else: - symmetric = False + version = f"{ATLAS_VERSION}.{atlas_minor_version}" + shape = reference_stack.shape + + # If no hemisphere file is given, assume the atlas is symmetric: + symmetric = hemispheres_stack is None # Check consistency of structures .json file: - structures = read_json(dir_path / descriptors.STRUCTURES_FILENAME) - check_struct_consistency(structures) + check_struct_consistency(structures_list) # Finalize metadata dictionary: metadata_dict = generate_metadata_dict( @@ -86,21 +80,32 @@ def wrapup_atlas_from_dir( shape=shape, ) - # write metadata dict: - with open(dir_path / descriptors.METADATA_FILENAME, "w") as f: - json.dump(metadata_dict, f) + atlas_dir_name = atlas_name + "_v" + version + dest_dir = working_dir / atlas_dir_name + dest_dir.mkdir() # exist_ok would be more permissive but error-prone here + + # save regions list json: + with open(dest_dir / descriptors.STRUCTURES_FILENAME, "w") as f: + json.dump(structures_list, f) + + # TODO use BGSpace and reorient stacks; + # TODO use BGSpace and reorient mesh; + # TODO find function to save meshes; + # write tiff stacks: + save_reference(reference_stack, dest_dir) + save_annotation(annotation_stack, dest_dir) - # Create human readable .csv and .txt files - create_metadata_files(dir_path, metadata_dict, structures, root) + # Create human readable .csv and .txt files: + create_metadata_files(dest_dir, metadata_dict, structures_list, root_id) # Compress if required: if compress: - output_filename = dir_path.parent / f"{dir_path.name}.tar.gz" + output_filename = dest_dir.parent / f"{dest_dir.name}.tar.gz" print(f"Saving compressed atlas data at: {output_filename}") with tarfile.open(output_filename, "w:gz") as tar: - tar.add(dir_path, arcname=dir_path.name) + tar.add(dest_dir, arcname=dest_dir.name) # Cleanup if required: if cleanup_files: # Clean temporary directory and remove it: - shutil.rmtree(dir_path) + shutil.rmtree(dest_dir) diff --git a/brainatlas_api/descriptors.py b/brainatlas_api/descriptors.py index 46dda7c..6f63bc9 100644 --- a/brainatlas_api/descriptors.py +++ b/brainatlas_api/descriptors.py @@ -35,3 +35,6 @@ REFERENCE_DTYPE = np.uint16 ANNOTATION_DTYPE = np.int32 + +# Standard orientation origin: Anterior, Left, Superior +ATLAS_ORIENTATION = "als"