/
base.py
141 lines (108 loc) · 4.19 KB
/
base.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
from abc import ABCMeta, abstractmethod, abstractproperty
from typing import Dict, Tuple
class Channel(metaclass=ABCMeta):
"""Channels are abstractions that enable ExecutionProviders to talk to
resource managers of remote compute facilities.
For certain resources such as campus clusters or supercomputers at
research laboratories, resource requirements may require authentication.
For instance some resources may allow access to their job schedulers from
only their login-nodes which require you to authenticate through SSH, or
require two factor authentication.
The simplest Channel, *LocalChannel*, executes commands locally in a
shell, while the *SSHChannel* authenticates you to remote systems.
Channels provide the ability to execute commands remotely, using the
execute_wait method, and manipulate the remote file system using methods
such as push_file, pull_file and makedirs.
Channels should ensure that each launched command runs in a new process
group, so that providers (such as AdHocProvider and LocalProvider) which
terminate long running commands using process groups can do so.
"""
@abstractmethod
def execute_wait(self, cmd: str, walltime: int = 0, envs: Dict[str, str] = {}) -> Tuple[int, str, str]:
''' Executes the cmd, with a defined walltime.
Args:
- cmd (string): Command string to execute over the channel
- walltime (int) : Timeout in seconds
KWargs:
- envs (Dict[str, str]) : Environment variables to push to the remote side
Returns:
- (exit_code, stdout, stderr) (int, string, string)
'''
pass
@abstractproperty
def script_dir(self) -> str:
''' This is a property. Returns the directory assigned for storing all internal scripts such as
scheduler submit scripts. This is usually where error logs from the scheduler would reside on the
channel destination side.
Args:
- None
Returns:
- Channel script dir
'''
pass
# DFK expects to be able to modify this, so it needs to be in the abstract class
@script_dir.setter
def script_dir(self, value: str) -> None:
pass
@abstractmethod
def push_file(self, source: str, dest_dir: str) -> str:
''' Channel will take care of moving the file from source to the destination
directory
Args:
source (string) : Full filepath of the file to be moved
dest_dir (string) : Absolute path of the directory to move to
Returns:
destination_path (string)
'''
pass
@abstractmethod
def pull_file(self, remote_source: str, local_dir: str) -> str:
''' Transport file on the remote side to a local directory
Args:
remote_source (string): remote_source
local_dir (string): Local directory to copy to
Returns:
destination_path (string)
'''
pass
@abstractmethod
def close(self) -> bool:
''' Closes the channel. Clean out any auth credentials.
Args:
None
Returns:
Bool
'''
pass
@abstractmethod
def makedirs(self, path: str, mode: int = 0o511, exist_ok: bool = False) -> None:
"""Create a directory.
If intermediate directories do not exist, they will be created.
Parameters
----------
path : str
Path of directory to create.
mode : int
Permissions (posix-style) for the newly-created directory.
exist_ok : bool
If False, raise an OSError if the target directory already exists.
"""
pass
@abstractmethod
def isdir(self, path: str) -> bool:
"""Return true if the path refers to an existing directory.
Parameters
----------
path : str
Path of directory to check.
"""
pass
@abstractmethod
def abspath(self, path: str) -> str:
"""Return the absolute path.
Parameters
----------
path : str
Path for which the absolute path will be returned.
"""
pass