begin packaging reference images

bg_atlasgen/component_gen/collate_upload_all.py

@@ -0,0 +1,36 @@
+import tempfile
+from pathlib import Path
+
+from bg_atlasapi import utils
+from fancylog import fancylog
+
+import bg_atlasgen as package_to_log
+from bg_atlasgen.component_gen.reference import collate_references
+from bg_atlasgen.utils.config import load_config
+
+TMPDIR_PREFIX = "bg_atlasgen_"
+DEBUG = True
+
+SAVE_DIR = Path.home() / "bg_atlasgen"
+
+
+def collate_all(save_dir, debug=False):
+    save_dir.mkdir(exist_ok=True, parents=True)
+    fancylog.start_logging(
+        save_dir,
+        package_to_log,
+        verbose=debug,
+        timestamp=True,
+    )
+
+    atlas_template = load_config("atlas_template.yaml")
+    utils.check_internet_connection()
+    tmp_path = Path(tempfile.mkdtemp(prefix=TMPDIR_PREFIX))
+    print(f"Created temporary directory: {tmp_path}")
+    collate_references.create_all_references(
+        tmp_path, atlas_template, debug=debug
+    )
+
+
+if __name__ == "__main__":
+    collate_all(SAVE_DIR, debug=DEBUG)

bg_atlasgen/component_gen/config/atlas_template.yaml

@@ -0,0 +1,34 @@
+remote_url_base: "https://gin.g-node.org/brainglobe/atlases/raw/master/{}"
+ATLAS_MAJOR_V: 0
+METADATA_TEMPLATE:
+  name: "name/author/institute_species_[optionalspecs]"
+  citation: "Someone et al 2020, https://doi.org/somedoi"
+  atlas_link: "http://www.example.com"
+  species: "Gen species"
+  symmetric: false
+  resolution: [1.0, 1.0, 1.0]
+  orientation: "asr"
+  shape: [100, 50, 100]
+  version: "0.0"
+  additional_references: []
+
+STRUCTURE_TEMPLATE:
+  acronym: "root"
+  id: 997
+  name: "root"
+  structure_id_path: [997]
+  rgb_triplet: [255, 255, 255]
+
+METADATA_FILENAME: "metadata.json"
+STRUCTURES_FILENAME: "structures.json"
+REFERENCE_FILENAME: "reference.tiff"
+ANNOTATION_FILENAME: "annotation.tiff"
+HEMISPHERES_FILENAME: "hemispheres.tiff"
+MESHES_DIRNAME: "meshes"
+
+REFERENCE_DTYPE: "uint16"
+ANNOTATION_DTYPE: "uint32"
+HEMISPHERES_DTYPE: "uint8"
+
+# Standard orientation origin: Anterior, Superior, Right (using BG-Space definition)
+ATLAS_ORIENTATION: "asr"

bg_atlasgen/component_gen/config/references.yaml

@@ -0,0 +1,14 @@
+allen_adult_mouse_stp:
+  resolution:
+    10: "http://download.alleninstitute.org/informatics-archive/current-release/mouse_ccf/average_template/average_template_10.nrrd"
+    25: "http://download.alleninstitute.org/informatics-archive/current-release/mouse_ccf/average_template/average_template_25.nrrd"
+    50: "http://download.alleninstitute.org/informatics-archive/current-release/mouse_ccf/average_template/average_template_50.nrrd"
+    100: "http://download.alleninstitute.org/informatics-archive/current-release/mouse_ccf/average_template/average_template_50.nrrd"
+  orientation: asl
+
+kim_dev_mouse_p56_lsfm:
+  resolution:
+    10: "https://data.mendeley.com/public-files/datasets/2svx788ddf/files/9777fb67-32f3-4c2b-85d7-421cecb15a5c/file_downloaded"
+    25: "https://data.mendeley.com/public-files/datasets/2svx788ddf/files/028d96bb-3793-424e-8e78-852df2788053/file_downloaded"
+    50: "https://data.mendeley.com/public-files/datasets/2svx788ddf/files/7d414c22-2f46-4205-8ab4-5a62cb547d94/file_downloaded"
+  orientation: asr

bg_atlasgen/component_gen/reference/collate_references.py

@@ -0,0 +1,50 @@
+import logging
+
+from bg_atlasgen.component_gen.reference import create_reference
+from bg_atlasgen.utils.config import load_config
+
+
+def create_all_references(tmp_path, atlas_template, debug=False):
+    reference_data = load_config("references.yaml")
+    for reference in reference_data:
+        logging.info(f"Creating reference images for: {reference}")
+        resolutions = reference_data[reference]["resolution"]
+
+        if debug:
+            # For speed, only run with one resolution
+            resolution = max(resolutions)
+            run_reference_generation(
+                tmp_path,
+                reference,
+                resolution,
+                resolutions[resolution],
+                reference_data[reference]["orientation"],
+                atlas_template,
+            )
+        else:
+            for resolution in resolutions:
+                run_reference_generation(
+                    tmp_path,
+                    reference,
+                    resolution,
+                    resolutions[resolution],
+                    reference_data[reference]["orientation"],
+                    atlas_template,
+                )
+
+
+def run_reference_generation(
+    tmp_path, reference, resolution, url, orientation, atlas_template
+):
+    logging.info(
+        f"Creating reference: {reference} at resolution: {resolution}"
+    )
+    function = getattr(create_reference, reference)
+    result = function(
+        tmp_path, reference, resolution, url, orientation, atlas_template
+    )
+    return result
+
+
+if __name__ == "__main__":
+    create_all_references()

bg_atlasgen/component_gen/reference/create_reference.py

@@ -0,0 +1,45 @@
+import logging
+
+import bg_space as bg
+import imio
+from bg_atlasapi import utils
+
+from bg_atlasgen.utils.image import write_to_compressed_tiff
+
+
+def allen_adult_mouse_stp(
+    temporary_directory, name, resolution, url, orientation, atlas_template
+):
+    filename = temporary_directory / f"{name}_{resolution}.nrrd"
+    download_and_resave_image(
+        url, filename, orientation, atlas_template["ATLAS_ORIENTATION"]
+    )
+
+
+def kim_dev_mouse_p56_lsfm(
+    temporary_directory, name, resolution, url, orientation, atlas_template
+):
+    filename = temporary_directory / f"{name}_{resolution}.nii.gz"
+    download_and_resave_image(
+        url, filename, orientation, atlas_template["ATLAS_ORIENTATION"]
+    )
+
+
+def download_and_resave_image(url, filename, orientation, final_orientation):
+    logging.debug(f"Downloading {url} to {filename}")
+    utils.retrieve_over_http(url, filename)
+    load_reorient_save(filename, orientation, final_orientation)
+
+
+def load_reorient_save(
+    filename, orientation, final_orientation, suffix=".tiff"
+):
+    logging.debug(f"Loading {filename}")
+    image = imio.load_any(filename)
+    logging.debug(
+        f"Reorienting image from {orientation} to {final_orientation}"
+    )
+    image = bg.map_stack_to(orientation, final_orientation, image)
+    output_filename = filename.with_suffix("").with_suffix(suffix)
+    logging.debug(f"Saving image to {output_filename}")
+    write_to_compressed_tiff(output_filename, image)

bg_atlasgen/utils/config.py

@@ -0,0 +1,11 @@
+from importlib.resources import open_text
+
+import yaml
+
+from bg_atlasgen.component_gen import config
+
+
+def load_config(yaml_file):
+    with open_text(config, yaml_file) as file:
+        data = yaml.safe_load(file)
+        return data

bg_atlasgen/utils/image.py

@@ -0,0 +1,5 @@
+from tifffile import imwrite
+
+
+def write_to_compressed_tiff(output_file, array_data, compression="deflate"):
+    imwrite(output_file, array_data, compression=compression)

pyproject.toml

@@ -19,6 +19,7 @@ classifiers = [
 ]
 requires-python = ">=3.8"
 dependencies = [
+    "fancylog",
     "PyMCubes",
     "SimpleITK",
     "bg_atlasapi",