Skip to content

Commit

Permalink
- preparing for test suite / introducing pep8 format checks
Browse files Browse the repository at this point in the history
  • Loading branch information
afabiani committed Nov 15, 2018
1 parent c2ca09b commit 8436d3a
Show file tree
Hide file tree
Showing 13 changed files with 170 additions and 160 deletions.
15 changes: 8 additions & 7 deletions src/wpsremote/computation_job_input.py
View file
Expand Up @@ -143,13 +143,14 @@ def as_json_string(self):
# {"type": "string", "description": "A persons surname", "max": 1, "default": "Meier"}
res = {}
attrib_to_convert = [
'_type',
'_title',
'_default',
'_description',
'_min',
'_max',
'_input_mime_type'] # missing _enum
'_type',
'_title',
'_default',
'_description',
'_min',
'_max',
'_input_mime_type'
] # missing _enum
attribute_list = [a for a in dir(self) if not a.startswith('__') and not callable(getattr(self, a))]
attribute_list_filtered = [x for x in attribute_list if x in attrib_to_convert]
for a in attribute_list_filtered:
Expand Down
18 changes: 14 additions & 4 deletions src/wpsremote/computation_job_inputs.py
View file
Expand Up @@ -51,12 +51,22 @@ def create_from_dict(paremeters_types_defs):
mininum = int(d['min']) if "min" in d else 1
maximum = int(d['max']) if "max" in d else 1
input_mime_type = d['input_mime_type'] if "input_mime_type" in d else None
input_to_add = computation_job_param.ComputationJobParam(
name, d['type'], title, d['description'], default, formatter, mininum, maximum, input_mime_type)
input_to_add = computation_job_param.ComputationJobParam(name,
d['type'],
title,
d['description'],
default,
formatter,
mininum,
maximum,
input_mime_type)
elif d['class'] == 'const':
title = d['title'] if "title" in d else None
input_to_add = computation_job_const.ComputationJobConst(
name, d['type'], title, d['description'], d['value'])
input_to_add = computation_job_const.ComputationJobConst(name,
d['type'],
title,
d['description'],
d['value'])
else:
raise TypeError("Unknown class value " + str(d['class']) + " for input " + str(name))
else:
Expand Down
15 changes: 4 additions & 11 deletions src/wpsremote/computation_job_param.py
View file
Expand Up @@ -13,17 +13,10 @@

class ComputationJobParam(computation_job_input.ComputationJobInput):

def __init__(
self,
name,
input_type,
title,
descr,
default=None,
formatter=None,
min_occurencies=0,
max_occurencies=1,
input_mime_type=None):
def __init__(self, name, input_type, title, descr,
default=None, formatter=None,
min_occurencies=0, max_occurencies=1,
input_mime_type=None):
super(ComputationJobParam, self).__init__(name, input_type, title, descr, default, formatter, input_mime_type)
self._min = min_occurencies
self._max = max_occurencies
Expand Down
6 changes: 2 additions & 4 deletions src/wpsremote/computational_job_input_action_update_json_file.py
View file
Expand Up @@ -24,13 +24,11 @@ def __init__(self, input_ref, target_json_file, json_path_expr, source_template_
self._input_ref = input_ref
# file path that will be updated
# self._target_json_file = target_json_file
self._target_json_file = target_json_file if isinstance(
target_json_file, path.path) else path.path(target_json_file)
self._target_json_file = target_json_file if isinstance(target_json_file, path.path) else path.path(target_json_file) # noqa
# Python-like expression to reference the attribute in the json file to be set
self.jsonpath_expr = json_path_expr
# If target file doesn't exists it will be created using the template at this file path
self._config_file_template = source_template_json_file if isinstance(
source_template_json_file, path.path) else path.path(source_template_json_file)
self._config_file_template = source_template_json_file if isinstance(source_template_json_file, path.path) else path.path(source_template_json_file) # noqa

def set_inputs(self, inputs):
if self._input_ref in inputs.names():
Expand Down
6 changes: 4 additions & 2 deletions src/wpsremote/computational_job_input_actions.py
View file
Expand Up @@ -22,7 +22,8 @@ class ComputationalJobInputActions(object):

@staticmethod
def create_from_config(input_sections):
'''Create a InputParameters object.
'''
Create a InputParameters object.
input_sections: a dictionary such as
{ action1 : [( 'par1_input1_name' , par1_input1_value ),
( 'par2_input1_name' , par2_input1_value ), ...],
Expand All @@ -38,7 +39,8 @@ def create_from_config(input_sections):

@staticmethod
def create_from_dict(input_action_def_dict):
'''Create a input parameters set from a definitions
'''
Create a input parameters set from a definitions
paremeters_types_defs is a dictionary such as
{input_ref1 : { 'action1_attrubute1_name' : action1_attrubute1_value, ... },
input_ref2 : { 'action2_attrubute1_name' : action2_attrubute1_value, ... }, ... }
Expand Down
9 changes: 4 additions & 5 deletions src/wpsremote/config_file_parameter_cmre_oaa.py
View file
Expand Up @@ -176,11 +176,10 @@ def test_OAA_main_config_file(self):
"template": "value"})]

paramValue = {'argx': 'test'}
paramValue = {
'numberOfEvaluations': 600,
'referenceDate': '20140901',
'timeHorizon': 259200,
'executiontType': 'oaaOnDemand'}
paramValue = {'numberOfEvaluations': 600,
'referenceDate': '20140901',
'timeHorizon': 259200,
'executiontType': 'oaaOnDemand'}

pip = input_parameters.InputParameters(paramsDef)
pip.parse(paramValue)
Expand Down
68 changes: 34 additions & 34 deletions src/wpsremote/output_file_parameter.py
View file
Expand Up @@ -60,24 +60,24 @@ def get_name(self):
def as_json_string(self):
# {"type": "string", "description": "A persons surname", "max": 1, "default": "Meier"}
res = {}
attrib_to_convert = [
'_type',
'_description',
'_title',
'_output_mime_type',
'_publish_as_layer',
'_publish_layer_name',
'_publish_default_style',
'_publish_target_workspace']
attrib_to_convert = ['_type',
'_description',
'_title',
'_output_mime_type',
'_publish_as_layer',
'_publish_layer_name',
'_publish_default_style',
'_publish_target_workspace']
attribute_list = [a for a in dir(self) if not a.startswith('__') and not callable(getattr(self, a))]
attribute_list_filtered = [x for x in attribute_list if x in attrib_to_convert]
for a in attribute_list_filtered:
res[a[1:]] = getattr(self, a)
return json.dumps(res)

def get_value(self):
if self._backup_on_wps_execution_shared_dir is not None and self._backup_on_wps_execution_shared_dir.lower(
) == "true" and self._wps_execution_shared_dir is not None:
if self._backup_on_wps_execution_shared_dir is not None and \
self._backup_on_wps_execution_shared_dir.lower() == "true" and \
self._wps_execution_shared_dir is not None:
unique_dirname = str(uuid.uuid4())
bkp_dir = path.path(self._wps_execution_shared_dir + "/" + unique_dirname)
bkp_dir.makedirs()
Expand Down Expand Up @@ -183,24 +183,24 @@ def get_name(self):
def as_json_string(self):
# {"type": "string", "description": "A persons surname", "max": 1, "default": "Meier"}
res = {}
attrib_to_convert = [
'_type',
'_description',
'_title',
'_output_mime_type',
'_publish_as_layer',
'_publish_layer_name',
'_publish_default_style',
'_publish_target_workspace']
attrib_to_convert = ['_type',
'_description',
'_title',
'_output_mime_type',
'_publish_as_layer',
'_publish_layer_name',
'_publish_default_style',
'_publish_target_workspace']
attribute_list = [a for a in dir(self) if not a.startswith('__') and not callable(getattr(self, a))]
attribute_list_filtered = [x for x in attribute_list if x in attrib_to_convert]
for a in attribute_list_filtered:
res[a[1:]] = getattr(self, a)
return json.dumps(res)

def get_value(self):
if self._backup_on_wps_execution_shared_dir is not None and self._backup_on_wps_execution_shared_dir.lower(
) == "true" and self._wps_execution_shared_dir is not None:
if self._backup_on_wps_execution_shared_dir is not None and \
self._backup_on_wps_execution_shared_dir.lower() == "true" and \
self._wps_execution_shared_dir is not None:
unique_dirname = str(uuid.uuid4())
bkp_dir = path.path(self._wps_execution_shared_dir + "/" + unique_dirname)
bkp_dir.makedirs()
Expand Down Expand Up @@ -328,25 +328,25 @@ def get_name(self):
def as_json_string(self):
# {"type": "string", "description": "A persons surname", "max": 1, "default": "Meier"}
res = {}
attrib_to_convert = [
'_type',
'_description',
'_title',
'_output_mime_type',
'_publish_as_layer',
'_publish_layer_name',
'_files_to_publish',
'_default_styles',
'_target_workspaces']
attrib_to_convert = ['_type',
'_description',
'_title',
'_output_mime_type',
'_publish_as_layer',
'_publish_layer_name',
'_files_to_publish',
'_default_styles',
'_target_workspaces']
attribute_list = [a for a in dir(self) if not a.startswith('__') and not callable(getattr(self, a))]
attribute_list_filtered = [x for x in attribute_list if x in attrib_to_convert]
for a in attribute_list_filtered:
res[a[1:]] = getattr(self, a)
return json.dumps(res)

def get_value(self):
if self._backup_on_wps_execution_shared_dir is not None and self._backup_on_wps_execution_shared_dir.lower(
) == "true" and self._wps_execution_shared_dir is not None:
if self._backup_on_wps_execution_shared_dir is not None and \
self._backup_on_wps_execution_shared_dir.lower() == "true" and \
self._wps_execution_shared_dir is not None:
unique_dirname = str(uuid.uuid4())
bkp_dir = path.path(self._wps_execution_shared_dir + "/" + unique_dirname)
bkp_dir.makedirs()
Expand Down
22 changes: 16 additions & 6 deletions src/wpsremote/output_parameters.py
View file
Expand Up @@ -47,21 +47,31 @@ def __init__(self, parameters_types_defs, wps_execution_shared_dir, uploader):
if ('type' in d):

if ('string' == d['type']):
self._params[name] = output_file_parameter.OutputFileParameter(
name, d, wps_execution_shared_dir=self._wps_execution_shared_dir, uploader=uploader)
self._params[name] = output_file_parameter.\
OutputFileParameter(name,
d,
wps_execution_shared_dir=self._wps_execution_shared_dir,
uploader=uploader)

elif ('image/geotiff' in d['type'] or
'text/xml' in d['type'] or
'application/gml' in d['type'] or
'application/zip' in d['type'] or
'application/x-netcdf' in d['type'] or
'video/mp4' in d['type']):
self._params[name] = output_file_parameter.RawFileParameter(
name, d, wps_execution_shared_dir=self._wps_execution_shared_dir, uploader=uploader)
self._params[name] = output_file_parameter.\
RawFileParameter(name,
d,
wps_execution_shared_dir=self._wps_execution_shared_dir,
uploader=uploader)

elif ('application/owc' in d['type']):
self._params[name] = output_file_parameter.OWCFileParameter(
name, d, parameters_types_defs, wps_execution_shared_dir=self._wps_execution_shared_dir, uploader=uploader)
self._params[name] = output_file_parameter.\
OWCFileParameter(name,
d,
parameters_types_defs,
wps_execution_shared_dir=self._wps_execution_shared_dir,
uploader=uploader)

else:
logger = logging.getLogger("OutputParameters.__init__")
Expand Down
21 changes: 12 additions & 9 deletions src/wpsremote/processbot.py
View file
Expand Up @@ -66,13 +66,13 @@ def __init__(self, remote_config_filepath, service_config_filepath, execute_mess

# the config file is read with raw=False because the unique_exe_id value
# will be used (interpolated) in the config
serviceConfig = configInstance.create(
service_config_filepath,
case_sensitive=True,
variables={
'unique_exe_id': self._uniqueExeId,
'wps_execution_shared_dir': self._wps_execution_shared_dir},
raw=False)
serviceConfig = configInstance.create(service_config_filepath,
case_sensitive=True,
variables={
'unique_exe_id': self._uniqueExeId,
'wps_execution_shared_dir': self._wps_execution_shared_dir
},
raw=False)

self.service = serviceConfig.get("DEFAULT", "service") # todo: what is?
self.namespace = serviceConfig.get("DEFAULT", "namespace")
Expand Down Expand Up @@ -107,8 +107,11 @@ def __init__(self, remote_config_filepath, service_config_filepath, execute_mess
rsa_key = RSA.importKey(privatekey, passphrase=uploader_passphrase)
uploader_password = rsa_key.decrypt(base64.b64decode(uploader_password))

self._uploader = introspection.get_class_four_arg(
uploader_class_name, uploader_host, uploader_username, uploader_password, self._uniqueExeId)
self._uploader = introspection.get_class_four_arg(uploader_class_name,
uploader_host,
uploader_username,
uploader_password,
self._uniqueExeId)
else:
self._uploader = None

Expand Down
22 changes: 9 additions & 13 deletions src/wpsremote/resource_cleaner.py
View file
Expand Up @@ -35,9 +35,7 @@ class Resource(object):

@staticmethod
def get_pid_files_dir():
return Resource.pid_files_dir if isinstance(
Resource.pid_files_dir, path.path) else path.path(
Resource.pid_files_dir)
return Resource.pid_files_dir if isinstance(Resource.pid_files_dir, path.path) else path.path(Resource.pid_files_dir) # noqa

@staticmethod
def clean_up_all():
Expand Down Expand Up @@ -195,7 +193,6 @@ def write(self):
self.filepath().remove()

config = ConfigParser.ConfigParser()

config.set("DEFAULT", "start_time", self.start_time_as_str() if self._start_time is not None else "")
config.set("DEFAULT", "unique_id", self._unique_id)
config.set("DEFAULT", "sendbox_path", str(self._sandbox_path))
Expand Down Expand Up @@ -261,10 +258,10 @@ def kill_process(self, pid, cmd_line):
logger.debug(traceback.format_exc())
return False
else:
logger.info(
"pid " +
str(pid) +
" found but with wrong command line. Assuming resource was already correcly cleaned-up")
logger.info("pid " +
str(pid) +
" found but with wrong command line. " +
"Assuming resource was already correcly cleaned-up")
return True
else:
logger.warning("pid " + str(pid) + " doesn't exist")
Expand Down Expand Up @@ -335,11 +332,10 @@ def test_workflow(self):
# srv bot
# create process bot
cmd = 'python -c "import time; time.sleep(80000)"'
invoked_process = subprocess.Popen(
args=cmd,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
invoked_process = subprocess.Popen(args=cmd,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
prcbot_pid = invoked_process.pid
unique_id = str(random.randint(1, 1000)).zfill(5)
sendbox_path = sendbox_root / str(unique_id)
Expand Down
10 changes: 6 additions & 4 deletions src/wpsremote/servicebot.py
View file
Expand Up @@ -57,9 +57,12 @@ def __init__(self, remote_config_filepath, service_config_filepath):
# (request hanlder); for example the unique execution id value to craete
# the sand box directory
self._service_config_file = service_config_filepath
serviceConfig = configInstance.create(
service_config_filepath, case_sensitive=True, variables={
'wps_execution_shared_dir': self._wps_execution_shared_dir}, raw=True)
serviceConfig = configInstance.create(service_config_filepath,
case_sensitive=True,
variables={
'wps_execution_shared_dir': self._wps_execution_shared_dir
},
raw=True)
self.service = serviceConfig.get("DEFAULT", "service") # WPS service name?
self.namespace = serviceConfig.get("DEFAULT", "namespace")
self.description = serviceConfig.get("DEFAULT", "description") # WPS service description
Expand Down Expand Up @@ -251,7 +254,6 @@ def output_parser_verbose(self, invoked_process, param_filepath):
try:
gs_UID = gs_UID_search.group(1)
gs_JID = gs_JID_search.group(1)

gs_MSG = gs_JID_search = re.search('<MSG>(.*)</MSG>', line, re.IGNORECASE).group(1)
except BaseException:
pass
Expand Down

0 comments on commit 8436d3a

Please sign in to comment.