WIP - adding the new resource parameter api to resources

dataworkspaces/resources/api_resource.py

@@ -31,20 +31,10 @@ def __init__(self, name:str, role:str, workspace:Workspace):
         super().__init__(API_RESOURCE_TYPE, name, role, workspace)
         self.hash_states = [] # type: List[Any]
 
-    def get_params(self) -> JSONDict:
-        return {
-            'resource_type':self.resource_type,
-            'name':self.name,
-            'role':self.role
-        }
-
     def validate_subpath_exists(self, subpath:str) -> None:
         raise ConfigurationError("Subpath %s is not valid for resource %s: API resources do not support subpaths"%
                                  (subpath, self.name))
 
-    def get_local_params(self) -> JSONDict:
-        return {}
-
     def get_local_path_if_any(self) -> Optional[str]:
         return None
 

dataworkspaces/resources/git_resource.py

@@ -10,7 +10,7 @@
 import stat
 import click
 import json
-from typing import Set, Pattern, Union, Optional, Tuple, Dict, Any
+from typing import Set, Pattern, Union, Optional, Tuple
 
 from dataworkspaces.errors import ConfigurationError, InternalError
 from dataworkspaces.utils.subprocess_utils import \
@@ -36,7 +36,7 @@
 from dataworkspaces.utils.file_utils \
     import LocalPathType, does_subpath_exist, get_subpath_from_absolute
 from dataworkspaces.utils.param_utils import \
-    ResourceParams, make_validate_by_type, validate_relpath, parse_bool,\
+    make_validate_by_type, validate_relpath, parse_bool,\
     validate_abspath
 
 from dataworkspaces.utils.snapshot_utils import move_current_files_local_fs
@@ -75,7 +75,6 @@ def __init__(self, resource_type:str, name:str, role:str, workspace:Workspace,
         super().__init__(resource_type, name, role, workspace)
 
         # define and validate the parameters
-        self.param_defs = ResourceParams()
         self.param_defs.define('export',
                                default_value=False,
                                optional=True,
@@ -98,12 +97,6 @@ def __init__(self, resource_type:str, name:str, role:str, workspace:Workspace,
     def get_local_path_if_any(self):
         return self.local_path
 
-    def get_params(self) -> Dict[str,Any]:
-        return self.param_defs.get_params(self)
-
-    def get_local_params(self) -> Dict[str,Any]:
-        return self.param_defs.get_local_params(self)
-
     def validate_subpath_exists(self, subpath:str) -> None:
         super().validate_subpath_exists(subpath)
 

dataworkspaces/resources/local_file_resource.py

@@ -21,6 +21,7 @@
 import dataworkspaces.resources.hashtree as hashtree
 from dataworkspaces.utils.snapshot_utils import move_current_files_local_fs
 import dataworkspaces.backends.git as git_backend
+from dataworkspaces.utils.param_utils import make_validate_by_type, parse_bool
 
 
 LOCAL_FILE = 'file'
@@ -35,12 +36,33 @@ def __init__(self, name:str, role:str, workspace:Workspace,
                  global_local_path:str, my_local_path:Optional[str],
                  ignore:List[str]=[], compute_hash:bool=False):
         super().__init__(LOCAL_FILE, name, role, workspace)
-        self.local_path = my_local_path if my_local_path is not None \
-                          else global_local_path
-        self.global_local_path = global_local_path
-        self.my_local_path = my_local_path
-        self.ignore = ignore
-        self.compute_hash = compute_hash
+        self.param_defs.define('global_local_path',
+                               default_value=None,
+                               optional=False,
+                               is_global=True,
+                               help="Location of files on local filesystem, as defined when the resource is created. "+
+                                    "May be overridden locally via my_local_path.",
+                               validation_fn=make_validate_by_type(str))
+        self.global_local_path = self.param_defs.get('global_local_path', global_local_path) # type: str
+        self.param_defs.define('my_local_path',
+                               default_value=None,
+                               optional=True,
+                               is_global=False,
+                               help="Override of global_local_path, just for this instance of the workspace.",
+                               validation_fn=make_validate_by_type(str))
+        self.my_local_path = self.param_defs.get('my_local_path', my_local_path) # type: Optional[str]
+        # the actual local path we'll use
+        self.local_path = self.my_local_path if self.my_local_path is not None \
+                          else self.global_local_path
+        self.param_defs.define('compute_hash',
+                               default_value=False,
+                               optional=True,
+                               is_global=True,
+                               help="If True, then compute the full hash of all files rather than using sizes.",
+                               validation_fn=make_validate_by_type(bool),
+                               parse_fn=parse_bool)
+        self.compute_hash = self.param_defs.get('compute_hash', compute_hash) # type: bool
+        self.ignore = ignore # TODO: should this be a parameter?
         if isinstance(workspace, git_backend.Workspace):
             # if the workspace is a git repo, then we can store our
             # hash files there.
@@ -205,7 +227,6 @@ def __str__(self):
 class LocalFileFactory(ResourceFactory):
     def from_command_line(self, role, name, workspace, local_path, export, compute_hash):
         """Instantiate a resource object from the add command's arguments"""
-        print("local_path=%s, export=%s, compute_hash=%s" % (local_path, export, compute_hash)) # XXX
         if not os.path.isdir(local_path):
             raise ConfigurationError(local_path + ' does not exist')
         if not os.access(local_path, os.R_OK): 
@@ -239,9 +260,11 @@ def from_json(self, params:JSONDict, local_params:JSONDict,
                   workspace:Workspace) -> LocalFileResource:
         """Instantiate a resource object from saved params and local params"""
         return LocalFileResource(params['name'], params['role'], workspace,
-                                 params['local_path'],
-                                 local_params['local_path'] if 'local_path' in local_params
-                                 else None,
+                                 # for backward compatibility, we also check for "local_path"
+                                 params['global_local_path'] if 'global_local_path' in params else params['local_path'],
+                                 local_params['my_local_path'] if 'my_local_path' in local_params
+                                 else (local_params['local_path'] if 'local_path' in local_params
+                                       else None),
                                  compute_hash=params['compute_hash'])
 
     def has_local_state(self) -> bool:
@@ -253,7 +276,8 @@ def clone(self, params:JSONDict, workspace:Workspace) -> LocalStateResourceMixin
         it is in th correct place.
         """
         name = params['name']
-        global_local_path = params['local_path'] # type: str
+        # check local_path, too for backward compatibility
+        global_local_path = params['global_local_path'] if 'global_local_path' in params else params['local_path'] # type: str
         local_params = {} # type: JSONDict
         if exists(global_local_path):
             local_path = global_local_path
@@ -264,7 +288,7 @@ def clone(self, params:JSONDict, workspace:Workspace) -> LocalStateResourceMixin
                          click.prompt("Local files resource '%s' was located at '%s' on the original system. Where is it located on this system?"%
                                       (name, global_local_path),
                                       type=LocalPathType(exists=True)))
-                local_params['local_path'] = local_path
+                local_params['my_local_path'] = local_path
             else:
                 raise ConfigurationError("Local files resource %s is missing from %s." % (name, global_local_path))
         if not isinstance(workspace, git_backend.Workspace):

dataworkspaces/workspace.py

@@ -66,7 +66,7 @@ def COMMAND_command(workspace, ...):
                                              ParamNotFoundError,\
                                              RESULTS_DIR_TEMPLATE,\
                                              RESULTS_MOVE_EXCLUDE_FILES,\
-                                             HOSTNAME
+                                             HOSTNAME, ResourceParams
 from dataworkspaces.utils.snapshot_utils import validate_template,\
                                                 expand_dir_template,\
                                                 make_re_pattern_for_dir_template
@@ -626,16 +626,17 @@ def __init__(self, resource_type:str, name:str, role:str, workspace:Workspace):
         self.role = role
         #: attribute: The workspace that contains this resource (Workspace)
         self.workspace = workspace
+        # setup our parameter definitions
+        self.param_defs = ResourceParams()
 
     def has_results_role(self):
         return self.role==ResourceRoles.RESULTS
 
-    @abstractmethod
     def get_params(self) -> JSONDict:
         """Get the parameters that define the configuration
         of the resource globally.
         """
-        pass
+        return self.param_defs.get_params(self)
 
     @abstractmethod
     def validate_subpath_exists(self, subpath:str) -> None:
@@ -702,12 +703,11 @@ class LocalStateResourceMixin(metaclass=ABCMeta):
     """Mixin for the resource api for resources with local state
     that need to be "cloned"
     """
-    @abstractmethod
     def get_local_params(self) -> JSONDict:
         """Get the parameters that define any local configuration of
         the resource (e.g. local filepaths)
         """
-        pass
+        return cast(Resource, self).param_defs.get_local_params(cast(Resource, self))
 
     @abstractmethod
     def get_local_path_if_any(self) -> Optional[str]: