Skip to content

Commit

Permalink
add new parameter typing; cleanup parameters for local file and rclone
Browse files Browse the repository at this point in the history
  • Loading branch information
@jfischer
jfischer committed Mar 27, 2020
1 parent d362c6c commit 098a1be
Show file tree
Hide file tree
Showing 5 changed files with 300 additions and 185 deletions.
2 changes: 1 addition & 1 deletion dataworkspaces/dws.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -298,7 +298,7 @@ def local_files(ctx, role, name, path, export:bool, compute_hash:bool):
@click.argument('source', type=str)
@click.argument('dest', type=str) # Currently, dest is required. Later: make dest optional and use the same path as remote?
@click.pass_context
def rclone(ctx, role, name, config, export, compute_hash, source, dest):
def rclone(ctx, role, name, config:str, export:bool, compute_hash:bool, source:str, dest:str):
"""Add an rclone-d repository as a resource to the workspace. Subcommand of ``add``"""
ns = ctx.obj
if role is None:
Expand Down
37 changes: 17 additions & 20 deletions dataworkspaces/resources/git_resource.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,7 @@
from dataworkspaces.utils.file_utils \
import LocalPathType, does_subpath_exist, get_subpath_from_absolute
from dataworkspaces.utils.param_utils import \
make_validate_by_type, validate_relpath, parse_bool,\
validate_abspath
BoolType, AbspathType, StringType, RelpathType

from dataworkspaces.utils.snapshot_utils import move_current_files_local_fs

Expand Down Expand Up @@ -80,16 +79,15 @@ def __init__(self, resource_type:str, name:str, role:str, workspace:Workspace,
optional=True,
help="True if metadata for export should be added each snapshot",
is_global=True,
parse_fn=parse_bool,
validation_fn=make_validate_by_type(bool))
ptype=BoolType())
self.export = self.param_defs.get('export', export) # type: bool
self.param_defs.define('local_path',
default_value=None,
optional=False,
help="Always points to the root of this resource "+
"(not necessarily the root of the repo)",
is_global=False,
validation_fn=validate_abspath)
ptype=AbspathType())
self.local_path = self.param_defs.get('local_path', local_path) # type: str

self.repo_dir = repo_dir # The root of the repo.
Expand Down Expand Up @@ -178,29 +176,28 @@ def __init__(self, name:str, role:str, workspace:Workspace,
optional=False,
help="URL of the remote git repo",
is_global=True,
validation_fn=make_validate_by_type(str))
ptype=StringType())
self.remote_origin_url = self.param_defs.get('remote_origin_url', remote_origin_url) # type: str
self.param_defs.define('relative_local_path',
default_value=None,
optional=True,
help="Local path of repo relative to the workspace",
is_global=True,
validation_fn=validate_relpath)
ptype=RelpathType())
self.relative_local_path = self.param_defs.get('relative_local_path', relative_local_path) # type: Optional[str]
self.param_defs.define('branch',
default_value=None,
optional=False,
help="Git branch to use",
is_global=True,
validation_fn=make_validate_by_type(str))
ptype=StringType())
self.branch = self.param_defs.get('branch', branch) # type: str
self.param_defs.define('read_only',
default_value=False,
optional=True,
help="If True, than no pushes are done for this repo",
is_global=True,
parse_fn=parse_bool,
validation_fn=make_validate_by_type(bool))
ptype=BoolType())
self.read_only = self.param_defs.get('read_only', read_only) # type: bool


Expand Down Expand Up @@ -472,8 +469,8 @@ def from_json(self, params, local_params, workspace):
workspace, params['remote_origin_url'],
params.get('relative_local_path', None),
local_path, params['branch'],
params.get('read_only'),
params.get('export'))
params.get('read_only', False),
params.get('export', False))

def has_local_state(self):
return True
Expand All @@ -487,7 +484,7 @@ def clone(self, params, workspace):
default_local_path = join(workspace_dir, relative_local_path) \
if relative_local_path else join(workspace_dir, rname)
branch = params['branch']
read_only = params.get('read_only')
read_only = params.get('read_only', False)
if not workspace.batch:
# ask the user for a local path
local_path = \
Expand Down Expand Up @@ -526,7 +523,7 @@ def clone(self, params, workspace):
return GitRepoResource(rname, params['role'],
workspace, remote_origin_url,
relative_local_path, local_path,
branch, read_only, params['export'])
branch, read_only, params.get('export', False))

def suggest_name(self, workspace, role, local_path, branch,
read_only, export):
Expand Down Expand Up @@ -559,7 +556,7 @@ def __init__(self, name:str, workspace:Workspace, relative_path:str, export:bool
optional=False,
help="Path of resource's directory relative to the workspace root",
is_global=True,
validation_fn=validate_relpath)
ptype=RelpathType())
self.relative_path = self.param_defs.get('relative_path', relative_path) # type: str

def results_move_current_files(self, rel_dest_root:str, exclude_files:Set[str],
Expand Down Expand Up @@ -665,7 +662,7 @@ def __init__(self, name:str, role:str, workspace:Workspace, relative_path:str, e
optional=False,
help="Path of resource's directory relative to the workspace root",
is_global=True,
validation_fn=validate_relpath)
ptype=RelpathType())
self.relative_path = self.param_defs.get('relative_path', relative_path) # type: str

def results_move_current_files(self, rel_dest_root:str, exclude_files:Set[str],
Expand Down Expand Up @@ -802,10 +799,10 @@ def from_json(self, params, local_params, workspace):
if params['role']==ResourceRoles.RESULTS:
return GitRepoResultsSubdirResource(params['name'],
workspace,
params['relative_path'], params['export'])
params['relative_path'], params.get('export', False))
else:
return GitRepoSubdirResource(params['name'], params['role'],
workspace, params['relative_path'], params['export'])
workspace, params['relative_path'], params.get('export', False))

def clone(self, params, workspace):
assert params['resource_type']=='git-subdirectory'
Expand All @@ -825,9 +822,9 @@ def clone(self, params, workspace):
# the contents, but will create a placeholder so our checks pass.
os.mkdir(local_path)
if role==ResourceRoles.RESULTS:
return GitRepoResultsSubdirResource(rname, workspace, relative_path, params['export'])
return GitRepoResultsSubdirResource(rname, workspace, relative_path, params.get('export', False))
else:
return GitRepoSubdirResource(rname, role, workspace, relative_path, params['export'])
return GitRepoSubdirResource(rname, role, workspace, relative_path, params.get('export', False))

def has_local_state(self) -> bool:
return True
Expand Down
34 changes: 15 additions & 19 deletions dataworkspaces/resources/local_file_resource.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
import dataworkspaces.resources.hashtree as hashtree
from dataworkspaces.utils.snapshot_utils import move_current_files_local_fs
import dataworkspaces.backends.git as git_backend
from dataworkspaces.utils.param_utils import make_validate_by_type, parse_bool
from dataworkspaces.utils.param_utils import StringType, BoolType


LOCAL_FILE = 'file'
Expand All @@ -34,33 +34,39 @@ def _relative_rsrc_dir_for_git_workspace(role, name):
class LocalFileResource(Resource, LocalStateResourceMixin, FileResourceMixin, SnapshotResourceMixin):
def __init__(self, name:str, role:str, workspace:Workspace,
global_local_path:str, my_local_path:Optional[str],
ignore:List[str]=[], compute_hash:bool=False):
export:bool, compute_hash:bool, ignore:List[str]=[]):
super().__init__(LOCAL_FILE, name, role, workspace)
self.param_defs.define('global_local_path',
default_value=None,
optional=False,
is_global=True,
help="Location of files on local filesystem, as defined when the resource is created. "+
"May be overridden locally via my_local_path.",
validation_fn=make_validate_by_type(str))
ptype=StringType())
self.global_local_path = self.param_defs.get('global_local_path', global_local_path) # type: str
self.param_defs.define('my_local_path',
default_value=None,
optional=True,
is_global=False,
help="Override of global_local_path, just for this instance of the workspace.",
validation_fn=make_validate_by_type(str))
ptype=StringType())
self.my_local_path = self.param_defs.get('my_local_path', my_local_path) # type: Optional[str]
# the actual local path we'll use
self.local_path = self.my_local_path if self.my_local_path is not None \
else self.global_local_path
self.param_defs.define('export',
default_value=False,
optional=True,
help="True if metadata for export should be added each snapshot",
is_global=True,
ptype=BoolType())
self.export = self.param_defs.get('export', export) # type: bool
self.param_defs.define('compute_hash',
default_value=False,
optional=True,
is_global=True,
help="If True, then compute the full hash of all files rather than using sizes.",
validation_fn=make_validate_by_type(bool),
parse_fn=parse_bool)
ptype=BoolType())
self.compute_hash = self.param_defs.get('compute_hash', compute_hash) # type: bool
self.ignore = ignore # TODO: should this be a parameter?
if isinstance(workspace, git_backend.Workspace):
Expand All @@ -79,15 +85,6 @@ def __init__(self, name:str, role:str, workspace:Workspace,
self.rsrcdir = os.path.abspath(os.path.join(self.local_path, '.hashes'))
self.ignore.append('.hashes')

def get_params(self) -> JSONDict:
return {
'resource_type':self.resource_type,
'name':self.name,
'role':self.role,
'local_path':self.global_local_path,
'compute_hash':self.compute_hash
}

def get_local_path_if_any(self):
return self.local_path

Expand Down Expand Up @@ -227,12 +224,12 @@ def __str__(self):
class LocalFileFactory(ResourceFactory):
def from_command_line(self, role, name, workspace, local_path, export, compute_hash):
"""Instantiate a resource object from the add command's arguments"""
workspace_path = workspace.get_workspace_local_path_if_any()
if not os.path.isdir(local_path):
raise ConfigurationError(local_path + ' does not exist')
if not os.access(local_path, os.R_OK):
raise ConfigurationError(local_path + ' does not have read permission')
if isinstance(workspace, git_backend.Workspace):
workspace_path = workspace.get_workspace_local_path_if_any()
assert workspace_path is not None
hash_path = join(workspace_path, _relative_rsrc_dir_for_git_workspace(role, name))
try:
Expand All @@ -254,7 +251,7 @@ def from_command_line(self, role, name, workspace, local_path, export, compute_h
if not exists(non_git_hashes):
os.mkdir(non_git_hashes)
return LocalFileResource(name, role, workspace, local_path, None,
compute_hash=compute_hash)
export=export, compute_hash=compute_hash)

def from_json(self, params:JSONDict, local_params:JSONDict,
workspace:Workspace) -> LocalFileResource:
Expand All @@ -265,7 +262,7 @@ def from_json(self, params:JSONDict, local_params:JSONDict,
local_params['my_local_path'] if 'my_local_path' in local_params
else (local_params['local_path'] if 'local_path' in local_params
else None),
compute_hash=params['compute_hash'])
export=params.get('export', False), compute_hash=params['compute_hash'])

def has_local_state(self) -> bool:
return True
Expand Down Expand Up @@ -298,6 +295,5 @@ def clone(self, params:JSONDict, workspace:Workspace) -> LocalStateResourceMixin
return self.from_json(params, local_params, workspace)

def suggest_name(self, workspace, role, local_path, export, compute_hash):
print("local_path=%s, export=%s, compute_hash=%s" % (local_path, export, compute_hash)) # XXX
return os.path.basename(local_path)

0 comments on commit 098a1be

Please sign in to comment.