249 option for accessing raw files using ssh connection (#257)

* rewrite instruments loaders - all .loader methods must now start with assigning self.name and running self._shutil_copy2 - fid is generated through running self.generate_fid() and fid is added to the Data instances using self.fid as argument. These changes are introduced to allow for accessing files on remote locations using e.g. sftp.

* add comments on what to do next

* fix bug in arbin_res.py

* fix bug in arbin_res.py

* update OtherPath so that it works also for edge-cases (not tested on linux yet)

* fix bug in posix reading of arbin_res.py

* rename copy method in base and start implementing function in core for copying from external

* start to put meat on the bones - but still some way to go

* start to put meat on the bones - but still some way to go

* start to put meat on the bones - but still some way to go

* update OtherPath to behave a bit nicer, and continue working on copying files from external source (copy_external_file).

* test OtherPath

* further improvements of OtherPath, include downloading capability as well as moving the reading parameter-file from init to prmreader (but still calling it in __init__) and add reading .env file

* using OtherPath.copy in instrument readers for making a copy

* using OtherPath.copy in instrument readers for making a copy

* using OtherPath.copy in instrument readers for making a copy

* add testing of environment vars and otherpath copy

* In CellpyCell, instead of directly running .loader (when running .from_raw), run .execute_loader instead (that in turn runs the .loader for you) - this makes it possible to do stuff before and after running the .loader method.

* set _is_db = True as class instance in arbin_sql and arbin_sql_7

* update settings for readthedocs

* update settings for readthedocs

* update settings for readthedocs

.readthedocs.yaml

@@ -0,0 +1,17 @@
+version: 2
+
+formats: all
+
+# Build documentation in the docs/ directory with Sphinx
+sphinx:
+    configuration: docs/conf.py
+
+build:
+    os: ubuntu-22.04
+    tools:
+        python: "3.9"
+
+conda:
+    environment: docs/environment_rtd.yml
+
+

cellpy/__init__.py

@@ -20,13 +20,8 @@
 
 logging.getLogger(__name__).addHandler(logging.NullHandler())
 
-try:
-    prmreader._read_prm_file(prmreader._get_prm_file())
-except FileNotFoundError:
-    warnings.warn("Could not find the config-file")
-except UserWarning:
-    warnings.warn("Could not read the config-file")
-
+init = prmreader.initialize
+init()
 
 get = cellreader.get
 __all__ = [
@@ -36,6 +31,7 @@
     "prms",
     "filefinder",
     "get",
+    "init",
     "ureg",
     "Q",
 ]

cellpy/exceptions.py

@@ -63,4 +63,6 @@ class NoDataFound(Error):
 
 class UnderDefined(Error):
     """Raised when trying something that requires you to set
-    a missing prm first"""
+    a missing prm on environment variable first"""
+
+    pass

cellpy/parameters/.cellpy_prms_default.conf

@@ -11,6 +11,7 @@ Paths:
   templatedir: cellpy_data/templates
   batchfiledir: cellpy_data/batchfiles
   instrumentdir: cellpy_data/instruments
+  env_file: .env_cellpy
 FileNames:
   file_name_format: YYYYMMDD_[NAME]EEE_CC_TT_RR
   raw_extension: res

cellpy/parameters/prmreader.py

@@ -11,6 +11,7 @@
 from pprint import pprint
 
 import box
+import dotenv
 import ruamel
 from ruamel.yaml import YAML
 from ruamel.yaml.error import YAMLError
@@ -29,8 +30,31 @@
 yaml = YAML()
 
 
+def initialize():
+    """initializes cellpy by reading the config file and the environment file"""
+    try:
+        _read_prm_file(_get_prm_file())
+        _load_env_file()
+    except FileNotFoundError:
+        warnings.warn("Could not find the config-file")
+    except UserWarning:
+        warnings.warn("Could not read the config-file")
+
+
+def _load_env_file():
+    """loads the environment file"""
+    env_file = pathlib.Path(prms.Paths.env_file)
+    env_file_in_user_dir = pathlib.Path.home() / prms.Paths.env_file
+    if env_file.is_file():
+        dotenv.load_dotenv(env_file)
+    elif env_file_in_user_dir.is_file():
+        dotenv.load_dotenv(env_file_in_user_dir)
+    else:
+        logging.debug("No .env file found")
+
+
 def get_user_name():
-    """get the user name of the current user (cross platform)"""
+    """get the username of the current user (cross-platform)"""
     return getpass.getuser()
 
 

cellpy/parameters/prms.py

@@ -22,7 +22,7 @@
 # locations etc. for reading custom parameters
 script_dir = os.path.abspath(os.path.dirname(__file__))
 cur_dir = os.path.abspath(os.path.dirname(sys.argv[0]))
-user_dir = os.path.expanduser("~")
+user_dir = Path.home()
 wdir = Path(cur_dir)
 
 
@@ -67,6 +67,7 @@ class PathsClass(CellPyConfig):
     batchfiledir: Union[Path, str] = wdir
     instrumentdir: Union[Path, str] = wdir
     db_filename: str = "cellpy_db.xlsx"  # used for simple excel db reader
+    env_file: Union[Path, str] = user_dir / ".env_cellpy"
 
 
 @dataclass

cellpy/readers/cellreader.py

@@ -64,6 +64,7 @@
 )
 
 from cellpy.readers.core import (
+    OtherPath,
     Data,
     FileID,
     identify_last_data_point,
@@ -531,7 +532,7 @@ def _set_instrument(self, instrument, **kwargs):
         self.loader_class = self.instrument_factory.create(instrument, **kwargs)
         self.raw_limits = self.loader_class.get_raw_limits()
         # ----- create the loader ------------------------
-        self.loader = self.loader_class.loader
+        self.loader = self.loader_class.loader_executor
 
     def set_instrument(
         self,
@@ -693,6 +694,7 @@ def set_cellpy_datadir(self, directory=None):
             return
         self.cellpy_datadir = directory
 
+    # TODO 249: update this so that it is aligned with OtherPaths etc
     def check_file_ids(self, rawfiles, cellpyfile, detailed=False):
         """Check the stats for the files (raw-data and cellpy hdf5).
 
@@ -742,7 +744,7 @@ def check_file_ids(self, rawfiles, cellpyfile, detailed=False):
 
     def _check_raw(self, file_names, abort_on_missing=False):
         """Get the file-ids for the res_files."""
-
+        # TODO 249: update this so that it is aligned with OtherPaths (accepts ssh etc)
         strip_file_names = True
         check_on = self.filestatuschecker
         if not self._is_listtype(file_names):
@@ -926,6 +928,9 @@ def loadcell(
         # TODO @jepe Make setting or prm so that it is possible to update only new data
         # TODO @jepe Allow passing handle to progress-bar or update a global progressbar
 
+        warnings.warn(
+            DeprecationWarning("loadcell is deprecated. Use cellpy.get instead.")
+        )
         logging.debug("Started cellpy.cellreader.loadcell ")
 
         if cellpy_file is None:
@@ -1220,7 +1225,7 @@ def from_raw(
             pre_processor_hook (callable): function that will be applied to the data within the loader.
             post_processor_hook (callable): function that will be applied to the
                 cellpy.Dataset object after initial loading.
-            is_a_file (bool): performs a is_file check if set to True.
+            is_a_file (bool): performs an is_file check if set to True.
 
         Keyword Args for merging:
             recalc (bool): set to false if you don't want cellpy to automatically shift cycle number
@@ -1272,8 +1277,10 @@ def from_raw(
         for file_name in self.file_names:
             logging.debug("loading raw file:")
             logging.debug(f"{file_name}")
-            if is_a_file and not Path(file_name).is_file():
-                raise NoDataFound(f"Could not find the file {file_name}")
+            # TODO 249: this needs an update:
+            if is_a_file and not OtherPath(file_name).is_external:
+                if not Path(file_name).is_file():
+                    raise NoDataFound(f"Could not find the file {file_name}")
 
             new_data = raw_file_loader(
                 file_name, pre_processor_hook=pre_processor_hook, **kwargs
@@ -2178,6 +2185,7 @@ def _convert2fid_table(self, cell):
             warnings.warn("seems you lost info about your raw-data (missing fids)")
         return fidtable
 
+    # TODO 249: update this so that it is aligned with OtherPaths (accepts ssh etc)
     def _convert2fid_list(self, tbl):
         # used when reading cellpy-file
         logging.debug("converting loaded fid-table to FileID object")