v0.5.0 fix mypy error core/init

2.1

@@ -0,0 +1,36 @@
+Transaction
+
+  Prefix: /home/SKhelifi/miniconda3/envs/odeon-test
+
+  Updating specs:
+
+   - lightning
+   - ca-certificates
+   - certifi
+   - openssl
+
+
+  Package                Version  Build         Channel                  Size
+───────────────────────────────────────────────────────────────────────────────
+  Install:
+───────────────────────────────────────────────────────────────────────────────
+
+  + lightning              2.2.1  pyhd8ed1ab_0  conda-forge/noarch     Cached
+  + lightning-utilities   0.11.0  pyhd8ed1ab_0  conda-forge/noarch       27kB
+  + pytorch-lightning      2.2.1  pyhd8ed1ab_0  conda-forge/noarch      440kB
+
+  Summary:
+
+  Install: 3 packages
+
+  Total download: 466kB
+
+───────────────────────────────────────────────────────────────────────────────
+
+
+Confirm changes: [Y/n]
+Looking for: ['lightning']
+
+
+Pinned packages:
+  - python 3.10.*

CHANGELOG.md

@@ -1,4 +1,9 @@
 # Changes
+
+## 0.4.0 (2024-03-20)
+ ***V.0.4.0 first stable version of core API with plugins***
+- Full oriented plugin architecture
+- Plugin system is stable
 ## 0.3.0 (2023-11-13)
  ***V.0.3.0 merge the new version of Odeon into the main code***
 - Old code has been totally removed excepted changelogs

VERSION

@@ -1 +1 @@
-0.3.0
+0.5.0

odeon/__init__.py

@@ -5,10 +5,10 @@
 
 from omegaconf import OmegaConf
 
-from .core.python_env import debug_mode
-from .core.logger import get_logger
-from .core.env import get_env, Env
 from .core.default_path import ODEON_ENV, ODEON_PATH
+from .core.env import Env, get_env
+from .core.logger import get_logger
+from .core.python_env import debug_mode
 
 __name__ = 'odeon'
 
@@ -25,37 +25,26 @@
 
 
 def store_resolver(store_name: str) -> str:
-    match store_name:
-        case 'user_config_store':
-            return str(ENV.user_config_store)
-        case 'user_artefact_store':
-            return str(ENV.user_artefact_store)
-        case 'user_model_store':
-            return str(ENV.user_model_store)
-        case 'user_dataset_store':
-            return str(ENV.user_dataset_store)
-        case 'user_test_store':
-            return str(ENV.user_test_store)
-        case 'user_delivery_store':
-            return str(ENV.user_delivery_store)
-        case 'user_log_store':
-            return str(ENV.user_log_store)
-        case 'config_store':
-            return str(ENV.config_store)
-        case 'artefact_store':
-            return str(ENV.artefact_store)
-        case 'model_store':
-            return str(ENV.model_store)
-        case 'dataset_store':
-            return str(ENV.dataset_store)
-        case 'test_store':
-            return str(ENV.test_store)
-        case 'delivery_store':
-            return str(ENV.delivery_store)
-        case 'log_store':
-            return str(ENV.log_store)
-        case _:
-            raise ValueError(f'the store {store_name} is not a valid store name')
+    store_mapping = {
+        'user_config': str(ENV.user_config_store),
+        'user_artefact': str(ENV.user_artefact_store),
+        'user_model': str(ENV.user_model_store),
+        'user_dataset': str(ENV.user_dataset_store),
+        'user_test': str(ENV.user_test_store),
+        'user_delivery': str(ENV.user_delivery_store),
+        'user_log': str(ENV.user_log_store),
+        'config': str(ENV.config_store),
+        'artefact': str(ENV.artefact_store),
+        'model': str(ENV.model_store),
+        'dataset': str(ENV.dataset_store),
+        'test': str(ENV.test_store),
+        'delivery': str(ENV.delivery_store),
+        'log': str(ENV.log_store)
+    }
+    if store_name in store_mapping:
+        return store_mapping[store_name]
+    else:
+        raise ValueError(f'the store {store_name} is not a valid store name')
 
 
 OmegaConf.register_new_resolver(name='odn.store', resolver=store_resolver)

odeon/core/__init__.py

@@ -1,10 +1,10 @@
-from .plugins.plugin import OdnPlugin, PluginMaturity
 from .app import APP_REGISTRY
+from .plugins.plugin import OdnPlugin, PluginMaturity
 from .registry import GENERIC_REGISTRY
 
 app_plugin = OdnPlugin(name='app_plugin',
                        author='samy KHELIFI-RICHARDS',
-                       plugin_maturity=str(PluginMaturity.DEVELOPMENT.value),
+                       plugin_maturity=PluginMaturity.DEVELOPMENT.value,
                        version='0.1',
                        elements={'app_registry': {'class': APP_REGISTRY,
                                  'registry': GENERIC_REGISTRY}}

odeon/core/env.py

@@ -5,20 +5,17 @@
 from typing import Any, Dict, List, Optional
 from uuid import uuid4
 
-from jsonargparse import set_config_read_mode, ArgumentParser
+from jsonargparse import ArgumentParser, set_config_read_mode
 
-from .default_path import ODEON_ENV
-from .io_utils import generate_yaml_with_doc
-from .default_path import ODEON_PATH
-from .io_utils import create_path_if_not_exists
+from .default_path import ODEON_ENV, ODEON_PATH
+from .io_utils import create_path_if_not_exists, generate_yaml_with_doc
 from .logger import get_logger
-from .plugins.plugin import OdnPlugin
+from .plugins.plugin import OdnPlugin, load_plugins
+from .python_env import debug_mode
 # from .logger introspection.py get_logger
 # from .types introspection.py PARSER
 from .singleton import Singleton
-from .types import PARAMS, URI, PARSER
-from .plugins.plugin import load_plugins
-from .python_env import debug_mode
+from .types import PARAMS, PARSER, URI
 
 # enable URL with Parser
 _URL_ENABLED: bool = True
@@ -72,8 +69,8 @@
                 Here is a complete example of how to configure an Odeon environment with your env.yml.
                 By default, your env.yml will be loaded from $HOME, but you can also
                 configure your path with the following environment variable: ODEON_INSTALL_PATH.
-                
-                
+
+
                 set_env_variables:
                   API_KEY: "abc123"
                   ANOTHER_VAR: "xyz789"
@@ -249,22 +246,19 @@ def __init__(self, config: Optional[EnvConf] = None):
         create_path_if_not_exists(self.user_artefact_store)
         create_path_if_not_exists(self.user_feature_store)
         create_path_if_not_exists(self.user_dataset_store)
-        create_path_if_not_exists(self.user_.model_store)
+        create_path_if_not_exists(self.user_model_store)
         create_path_if_not_exists(self.user_test_store)
         create_path_if_not_exists(self.user_delivery_store)
         create_path_if_not_exists(self.user_log_store)
 
-
-
-
     def _create_user_path(self) -> URI | None:
         p: URI | None = None
         if self.project is not None:
             if self.project.name != '':
                 p = Path(str(self.project.name))
         if self.team is not None:
             if self.team.name != '':
-                p = Path(str(self.project.name)) if p is None else Path(p) / str(self.project.name)
+                p = Path(str(self.team.name)) if p is None else Path(p) / str(self.team.name)
         if self.user is not None:
             if self.user.name != '':
                 p = Path(str(self.user.name)) if p is None else Path(p) / str(self.user.name)

odeon/core/python_env.py

@@ -13,8 +13,11 @@ def is_running_from_ipython():
 
 
 def is_running_in_jupyter_notebook():
+    """
+    Returns
+    -------
+    """
     # TODO simplify code between functions is_running_in_jupyter_notebook and is_running_from_ipython
-
     if is_running_from_ipython() is False:
         return False
     try:
@@ -34,6 +37,7 @@ def is_running_in_jupyter_notebook():
 
 def get_debug_mode() -> bool:
     """
+
     Returns
     -------
      bool

odeon/data/data_module.py

@@ -1,6 +1,7 @@
 from dataclasses import dataclass
 from typing import Callable, Dict, List, Optional
 
+from pytorch_lightning.utilities import CombinedLoader
 from pytorch_lightning.utilities.types import (EVAL_DATALOADERS,
                                                TRAIN_DATALOADERS)
 from torch.utils.data import DataLoader, Dataset
@@ -40,8 +41,8 @@ class Input(OdnData):
     def __init__(self,
                  fit_params: List[Dict] | Dict | None = None,
                  validate_params: List[Dict] | Dict | None = None,
-                 test_params: Dict = None,
-                 predict_params: Dict = None):
+                 test_params: List[Dict] | Dict | None = None,
+                 predict_params: List[Dict] | Dict | None = None):
 
         super(Input, self).__init__()
         self.fit_params = fit_params
@@ -50,8 +51,8 @@ def __init__(self,
         self.predict_params = predict_params
         self._fit: Data | Dict[str, Data] | None = None
         self._validate: Data | Dict[str, Data] | None = None
-        self._test: Data | None = None
-        self._predict: Data | None = None
+        self._test: Data | Dict[str, Data] | None = None
+        self._predict: Data | Dict[str, Data] | None = None
 
     def setup(self, stage: Optional[str] = None) -> None:
         if stage == Stages.FIT.value or stage == Stages.FIT:
@@ -77,13 +78,18 @@ def setup(self, stage: Optional[str] = None) -> None:
                 if isinstance(self.validate_params, List)\
                 else Input._instantiate_data(params=dict(self.validate_params), stage=Stages.VALIDATE)
         if stage == Stages.TEST.value or stage == Stages.TEST:
-            assert self.test_params, f'you want to run a stage {stage} but you have not filled {stage}_params :' \
-                                     f'{self.test_params}'
-            self._test = Input._instantiate_data(params=self.test_params, stage=Stages.TEST)
+            assert self.test_params, (f'you want to run a stage {stage} but you have not filled {stage}_params :'
+                                      f'{self.test_params}')
+            self._test = {f'test-{i + 1}': Input._instantiate_data(params=params, stage=Stages.TEST) for i, params
+                          in enumerate(list(self.test_params))} if isinstance(self.test_params, List)\
+                else Input._instantiate_data(params=dict(self.test_params), stage=Stages.TEST)
         if stage == Stages.PREDICT.value or stage == Stages.PREDICT:
             assert self.predict_params, f'you want to run a stage {stage} but you have not filled {stage}_params :' \
                                         f'{self.predict_params}'
-            self._predict = Input._instantiate_data(params=self.predict_params, stage=Stages.PREDICT)
+            self._predict = {f'predict-{i + 1}': Input._instantiate_data(params=params, stage=Stages.PREDICT)
+                             for i, params in enumerate(list(self.predict_params))} \
+                if isinstance(self.predict_params, List) else Input._instantiate_data(params=dict(self.predict_params),
+                                                                                      stage=Stages.PREDICT)
 
     @staticmethod
     def _instantiate_data(params: Dict, stage: STAGES_OR_VALUE) -> Data:
@@ -113,33 +119,41 @@ def predict(self) -> Data | None:
         return self._predict
 
     def train_dataloader(self) -> TRAIN_DATALOADERS:
-        if isinstance(self._fit, Data):
+        if isinstance(self._fit, Data | dict):
             if isinstance(self._fit, Dict):
                 d: Dict[str, DataLoader] = {k: v.dataloader for k, v in self._fit.items()}
-                return d
+                return CombinedLoader(d, mode='min_size')
             else:
                 return self._fit.dataloader
         else:
             raise NotImplementedError('fit is not implemented yet, fit is None')
 
     def val_dataloader(self) -> EVAL_DATALOADERS:
-        if isinstance(self._validate, Data):
+        if isinstance(self._validate, Data | dict):
             if isinstance(self._validate, Dict):
-                l: List = [v.dataloader for k, v in self._validate.items()]
-                return l
+                d: Dict[str, DataLoader] = {k: v.dataloader for k, v in self._validate.items()}
+                return CombinedLoader(d, mode='min_size')
             else:
                 return self._validate.dataloader
         else:
             raise NotImplementedError('validate is not implemented yet, validate is None')
 
     def test_dataloader(self) -> EVAL_DATALOADERS:
-        if isinstance(self._test, Data):
-            return self._test.dataloader
+        if isinstance(self._test, Data | dict):
+            if isinstance(self._test, Dict):
+                d: Dict[str, DataLoader] = {k: v.dataloader for k, v in self._test.items()}
+                return CombinedLoader(d, mode='min_size')
+            else:
+                return self._test.dataloader
         else:
             raise NotImplementedError('test is not implemented yet, test is None')
 
     def predict_dataloader(self) -> EVAL_DATALOADERS:
-        if isinstance(self._predict, Data):
-            return self._predict.dataloader
+        if isinstance(self._predict, Data | dict):
+            if isinstance(self._predict, Dict):
+                d: Dict[str, DataLoader] = {k: v.dataloader for k, v in self._predict.items()}
+                return CombinedLoader(d, mode='min_size')
+            else:
+                return self._predict.dataloader
         else:
             raise NotImplementedError('predict is not implemented yet, predict is None')

odeon/models/change/arch/change_unet.py

@@ -1,19 +1,19 @@
 # Adapted code for Odeon, Originally from https://github.com/microsoft/torchgeo/blob/main/torchgeo
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # Licensed under the MIT License.
-
 """Fully convolutional change detection (FCCD) implementations."""
-from typing import Any, Callable, Dict, Optional, Sequence, Union, List
+
 from copy import deepcopy
+from typing import Any, Callable, Dict, Optional, Sequence, Union
 
+import einops
 # from typing introspection.py List
 import segmentation_models_pytorch as smp
 import torch
 from segmentation_models_pytorch import Unet
 from segmentation_models_pytorch.base.model import SegmentationModel
+from segmentation_models_pytorch.decoders.unet.decoder import UnetDecoder
 from torch import Tensor
-import einops
-
 
 Unet.__module__ = "segmentation_models_pytorch"
 
@@ -83,7 +83,6 @@ def __init__(
         encoder_out_channels = [c * 2 for c in self.encoder.out_channels[1:]]
         encoder_out_channels.insert(0, self.encoder.out_channels[0])
 
-        UnetDecoder = Unet.decoder.UnetDecoder
         self.decoder = UnetDecoder(
             encoder_channels=encoder_out_channels,
             decoder_channels=decoder_channels,
@@ -179,7 +178,7 @@ def __init__(self,
                  **kwargs: Any) -> None:
 
         kwargs["aux_params"] = None
-        decoder_channels: List[int] = list(DEFAULT_DECODER_CHANNELS) if decoder_channels is None \
+        decoder_channels = list(DEFAULT_DECODER_CHANNELS) if decoder_channels is None \
             else list(decoder_channels)
         super().__init__(encoder_name,
                          encoder_depth,
@@ -409,7 +408,7 @@ def __init__(self,
                                encoder_weights=str(trunk_params['encoder_weights']),
                                in_channels=int(trunk_params['in_channels']),
                                classes=int(trunk_params['classes']))
-            
+
             trunk_model.load_state_dict(torch.load(str(segmentation_trunk)))
 
             self.segmentation_trunk = trunk_model.encoder
@@ -419,13 +418,12 @@ def __init__(self,
         self.pseudo_siamese = pseudo_siamese
 
         if self.pseudo_siamese:  # deep copy if none segmentation trunk at init
-            if self.segmentation_trunk is None:
+            if segmentation_trunk is None:
                 self.segmentation_trunk = deepcopy(self.encoder)
         else:
-
-            if self.segmentation_trunk:
-
+            if segmentation_trunk:
                 self.change_head = self.segmentation_trunk
+
     def build(self):
         ...