Atlas version control - first draft (#44)

* Atlas version control - first draft

* Interaction with atlas version conf file in GIN

* Added some tests

atlas_gen/wrapup.py

@@ -19,7 +19,7 @@
 
 # This should be changed every time we make changes in the atlas
 # structure:
-ATLAS_VERSION = 0
+ATLAS_VERSION = descriptors.ATLAS_MAJOR_V
 
 
 def wrapup_atlas_from_data(

brainatlas_api/__init__.py

@@ -7,9 +7,8 @@
 
 
 def get_atlas_class_from_name(name):
-    names = [
-        f"{atlas.atlas_name}_v{atlas.version}" for atlas in available_atlases
-    ]
+    names = [atlas.atlas_name for atlas in available_atlases]
+
     atlases = {n: a for n, a in zip(names, available_atlases)}
 
     if name in atlases.keys():

brainatlas_api/bg_atlas.py

@@ -18,6 +18,10 @@
 ]
 
 
+def _version_tuple_from_str(version_str):
+    return tuple([int(n) for n in version_str.split(".")])
+
+
 class BrainGlobeAtlas(core.Atlas):
     """Add download functionalities to Atlas class.
 
@@ -31,16 +35,16 @@ class BrainGlobeAtlas(core.Atlas):
         """
 
     atlas_name = None
-    version = None
     _remote_url_base = (
-        "https://gin.g-node.org/brainglobe/atlases/raw/master/{}.tar.gz"
+        "https://gin.g-node.org/brainglobe/atlases/raw/master/{}"
     )
 
     def __init__(self, brainglobe_dir=None, interm_download_dir=None):
+        # Read BrainGlobe configuration file:
         conf = config.read_config()
 
-        # Use either input values or values from the config file, and create
-        # directory if it does not exist:
+        # Use either input locations or locations from the config file,
+        # and create directory if it does not exist:
         for dir, dirname in zip(
             [brainglobe_dir, interm_download_dir],
             ["brainglobe_dir", "interm_download_dir"],
@@ -53,26 +57,64 @@ def __init__(self, brainglobe_dir=None, interm_download_dir=None):
             dir_path.mkdir(exist_ok=True)
             setattr(self, dirname, dir_path)
 
-        try:
-            super().__init__(self.brainglobe_dir / self.atlas_full_name)
-
-        except FileNotFoundError:
-            print(
-                0, f"{self.atlas_full_name} not found locally. Downloading..."
-            )
+        # Look for this atlas in local brainglobe folder:
+        if self.local_full_name is None:
+            print(0, f"{self.atlas_name} not found locally. Downloading...")
             self.download_extract_file()
 
-            super().__init__(self.brainglobe_dir / self.atlas_full_name)
+        # Instantiate after eventual download:
+        super().__init__(self.brainglobe_dir / self.local_full_name)
+
+    @property
+    def local_version(self):
+        """If atlas is local, return actual version of the downloaded files;
+        Else, return none.
+        """
+        full_name = self.local_full_name
+
+        if full_name is None:
+            return None
+
+        return _version_tuple_from_str(full_name.split("_v")[-1])
+
+    @property
+    def remote_version(self):
+        """Remote version read from GIN conf file.
+        """
+        remote_url = self._remote_url_base.format("last_versions.conf")
+        versions_conf = utils.conf_from_url(remote_url)
+
+        return _version_tuple_from_str(
+            versions_conf["atlases"][self.atlas_name]
+        )
 
     @property
-    def atlas_full_name(self):
-        return f"{self.atlas_name}_v{self.version}"
+    def local_full_name(self):
+        """As we can't know the local version a priori, search candidate dirs
+        using name and not version number. If none is found, return None
+        """
+        pattern = f"{self.atlas_name}_v*"
+        candidate_dirs = list(self.brainglobe_dir.glob(pattern))
+
+        # If multiple folders exist, raise error:
+        if len(candidate_dirs) > 1:
+            raise FileExistsError(
+                f"Multiple versions of atlas {self.atlas_name} in {self.brainglobe_dir}"
+            )
+        # If no one exist, return None:
+        elif len(candidate_dirs) == 0:
+            return None
+        # Else, return actual name:
+        else:
+            return candidate_dirs[0].name
 
     @property
     def remote_url(self):
         """Format complete url for download.
         """
-        return self._remote_url_base.format(self.atlas_full_name)
+        maj, min = self.remote_version
+        name = f"{self.atlas_name}_v{maj}.{min}.tar.gz"
+        return self._remote_url_base.format(name)
 
     def download_extract_file(self):
         """Download and extract atlas from remote url.
@@ -95,36 +137,29 @@ def download_extract_file(self):
 
 class ExampleAtlas(BrainGlobeAtlas):
     atlas_name = "example_mouse_100um"
-    version = "0.2"
 
 
 class FishAtlas(BrainGlobeAtlas):
     atlas_name = "mpin_zfish_1um"
-    version = "0.2"
 
 
 class RatAtlas(BrainGlobeAtlas):
     # TODO fix hierarchy and meshes
     atlas_name = "ratatlas"
-    version = "0.1"
 
 
 class AllenBrain25Um(BrainGlobeAtlas):
     atlas_name = "allen_mouse_25um"
-    version = "0.2"
 
 
 class KimUnified25Um(BrainGlobeAtlas):
     atlas_name = "kim_unified_25um"
-    version = "0.1"
 
 
 class KimUnified50Um(BrainGlobeAtlas):
     atlas_name = "kim_unified_50um"
-    version = "0.1"
 
 
 class AllenHumanBrain500Um(BrainGlobeAtlas):
     # TODO fix meshes
     atlas_name = "allen_human_500um"
-    version = "0.1"

brainatlas_api/descriptors.py

@@ -1,5 +1,8 @@
 import numpy as np
 
+# Major version of atlases used by current brainatlas-api release:
+ATLAS_MAJOR_V = 0
+
 # Entries and types from this template will be used to check atlas info
 # consistency. Please keep updated both this and the function when changing
 # the structure.
@@ -16,6 +19,8 @@
     "supplementary_stacks": [],
 }
 
+
+# Template for a structure dictionary:
 STRUCTURE_TEMPLATE = {
     "acronym": "root",
     "id": 997,
@@ -25,16 +30,18 @@
 }
 
 
+# File and directory names for the atlas package:
 METADATA_FILENAME = "metadata.json"
 STRUCTURES_FILENAME = "structures.json"
 REFERENCE_FILENAME = "reference.tiff"
 ANNOTATION_FILENAME = "annotation.tiff"
 HEMISPHERES_FILENAME = "hemispheres.tiff"
 MESHES_DIRNAME = "meshes"
 
+# Types for the atlas stacks:
 REFERENCE_DTYPE = np.uint16
 ANNOTATION_DTYPE = np.uint32
 HEMISPHERES_DTYPE = np.uint8
 
-# Standard orientation origin: Anterior, Left, Superior
+# Standard orientation origin: Anterior, Left, Superior (using BGSpace definition)
 ATLAS_ORIENTATION = "asl"

brainatlas_api/list_atlases.py

@@ -33,7 +33,7 @@ def list_atlases():
         cls for cls in map(bg_atlas.__dict__.get, bg_atlas.__all__)
     ]
     for atlas in available_atlases:
-        name = f"{atlas.atlas_name}_v{atlas.version}"
+        name = f"{atlas.atlas_name}_v{atlas.local_version}"
         if name not in atlases.keys():
             atlases[str(name)] = dict(
                 downloaded=False,

brainatlas_api/utils.py

@@ -4,6 +4,7 @@
 import requests
 from tqdm.auto import tqdm
 import logging
+import configparser
 
 logging.getLogger("urllib3").setLevel(logging.WARNING)
 
@@ -28,7 +29,7 @@ def check_internet_connection(
         if not raise_error:
             print("No internet connection available.")
         else:
-            raise ValueError(
+            raise ConnectionError(
                 "No internet connection, try again when you are connected to the internet."
             )
     return False
@@ -56,6 +57,14 @@ def retrieve_over_http(url, output_file_path):
         )
 
 
+def conf_from_url(url):
+    text = requests.get(url).text
+    config = configparser.ConfigParser()
+    config.read_string(text)
+
+    return config
+
+
 # --------------------------------- File I/O --------------------------------- #
 def read_json(path):
     with open(path, "r") as f:

tests/conftest.py

@@ -4,6 +4,11 @@
 # import tempfile
 
 
+@pytest.fixture()
+def atlas():
+    return ExampleAtlas()
+
+
 @pytest.fixture(scope="module")
 def atlas_path():
     # brainglobe_path=tempfile.mkdtemp()

tests/test_bg_atlas.py

@@ -0,0 +1,30 @@
+import pytest
+import tempfile
+import shutil
+from brainatlas_api.bg_atlas import ExampleAtlas
+
+
+def test_versions(atlas):
+    assert atlas.local_version == atlas.remote_version
+
+
+def test_local_search():
+    brainglobe_dir = tempfile.mkdtemp()
+    interm_download_dir = tempfile.mkdtemp()
+
+    atlas = ExampleAtlas(
+        brainglobe_dir=brainglobe_dir, interm_download_dir=interm_download_dir
+    )
+
+    assert atlas.atlas_name in atlas.local_full_name
+
+    # Make a copy:
+    copy_filename = atlas.root_dir.parent / (atlas.root_dir.name + "_2")
+    shutil.copytree(atlas.root_dir, copy_filename)
+
+    with pytest.raises(FileExistsError) as error:
+        _ = ExampleAtlas(brainglobe_dir=brainglobe_dir)
+    assert "Multiple versions of atlas" in str(error)
+
+    shutil.rmtree(brainglobe_dir)
+    shutil.rmtree(interm_download_dir)

tests/test_atlas.py → tests/test_core_atlas.py

@@ -2,13 +2,6 @@
 
 import numpy as np
 
-from brainatlas_api.bg_atlas import ExampleAtlas
-
-
-@pytest.fixture()
-def atlas():
-    return ExampleAtlas()
-
 
 def test_initialization(atlas):
     assert atlas.metadata == {
@@ -69,10 +62,15 @@ def test_meshfile_from_id(atlas):
         atlas.meshfile_from_structure("CH")
         == atlas.root_dir / "meshes/567.obj"
     )
+    assert atlas.root_meshfile() == atlas.root_dir / "meshes/997.obj"
 
 
 def test_mesh_from_id(atlas):
-    # TODO will change depending on mesh loading package
     mesh = atlas.structures[567]["mesh"]
     assert np.allclose(mesh.points[0], [8019.52, 3444.48, 507.104])
-    assert np.allclose(mesh.cells[0].data[0], [0, 1, 2])
+
+    mesh = atlas.mesh_from_structure(567)
+    assert np.allclose(mesh.points[0], [8019.52, 3444.48, 507.104])
+
+    mesh = atlas.root_mesh()
+    assert np.allclose(mesh.points[0], [7896.56, 3384.15, 503.781])

tests/test_list_atlases.py

@@ -1,15 +1,17 @@
 import brainatlas_api
+import pytest
 
 
 def test_list_atlases():
     brainatlas_api.list_atlases()
 
 
-def test_get_atlas_from_name():
-    a1 = brainatlas_api.get_atlas_class_from_name("allen_mouse_25um_v0.2")
-    a2 = brainatlas_api.get_atlas_class_from_name("xxxx")
-
-    if a1 is None:
-        raise ValueError
-    if a2 is not None:
-        raise ValueError
+@pytest.mark.parametrize(
+    "key, is_none", [("allen_mouse_25um", False), ("xxx", True)]
+)
+def test_get_atlas_from_name(key, is_none):
+    a = brainatlas_api.get_atlas_class_from_name(key)
+    if is_none:
+        assert a is None
+    else:
+        assert a is not None